Author: mircea.markus Date: 2009-02-04 09:19:55 -0500 (Wed, 04 Feb 2009) New Revision: 7637 Modified: core/trunk/src/test/java/org/jboss/cache/integration/websession/BuddyReplicationFailoverTest.java core/trunk/src/test/java/org/jboss/cache/integration/websession/util/SessionManager.java core/trunk/src/test/java/org/jboss/cache/integration/websession/util/SessionMetadata.java core/trunk/src/test/java/org/jboss/cache/integration/websession/util/WebSessionTestBase.java core/trunk/src/test/java/org/jboss/cache/util/internals/replicationlisteners/ReplicationListener.java core/trunk/src/test/resources/configs/integration/web-session-cache-configs.xml Log: enhanced logging Modified: core/trunk/src/test/java/org/jboss/cache/integration/websession/BuddyReplicationFailoverTest.java =================================================================== --- core/trunk/src/test/java/org/jboss/cache/integration/websession/BuddyReplicationFailoverTest.java 2009-02-04 11:19:35 UTC (rev 7636) +++ core/trunk/src/test/java/org/jboss/cache/integration/websession/BuddyReplicationFailoverTest.java 2009-02-04 14:19:55 UTC (rev 7637) @@ -48,8 +48,27 @@ /** * @author Brian Stansberry * + * This test is disabled because of following: + * There seem to be some assumptions in the above tests that do not look right to me. + * I have an example I've investigated(logs etc), haven't studied all possible failure scenarios, though I can imagine some others. + * One of them is the next one, on testInvalidateOnFailoverToBackup: + + * This is the code I am talking about: + * InvalidationServlet invs = new InvalidationServlet(); + * MultipleActionServlet mas = new MultipleActionServlet(sas, invs); + * (....) + * req = new Request(mgr0, sessionId, mas); + * *1+2* req.execute(); (...) + * *3*BuddyReplicationAssertions.assertUnrelated(contextHostName, sessionId, mgr0.getCache()); + + *And some explanation + + *1) manager0.invalidatesSession (i.e. cache0.remove) + *2) manager0 tries to data gravitate session (i.e. cache0.get with forceDataGrav set to true). This is done last line in Request.execute(). + *3) asserts that session is no longer present on cache0 + + *Now, the assumption at 3 is not necessarily valid. With some funny threading and using ASYNC replication, you might have the remote removeNode command originated from cache0 to run *after* gravitateData originated at step2. I.e. RemoveNodeCommand being executed after GravitateDataCommand on cache*1* (buddy of cache0). */ -//todo - fix intermitent failures @Test(groups = "integration",enabled = false, testName = "integration.websession.BuddyReplicationFailoverTest") public class BuddyReplicationFailoverTest extends WebSessionTestBase { @@ -521,7 +540,7 @@ // Create the session SetAttributesServlet sas = new SetAttributesServlet(Collections.singletonMap(KEY, getAttributeValue(attr++))); - ReplicationListener replListener0 = getReplicationListener(mgr0.getCache()); + ReplicationListener replListener0 = replListeners[0]; replListener0.expectWithTx(PutDataMapCommand.class); Request req = new Request(mgr3, null, sas); @@ -559,7 +578,7 @@ sas = new SetAttributesServlet(Collections.singletonMap(KEY, getAttributeValue(attr++))); InvalidationServlet invs = new InvalidationServlet(); MultipleActionServlet mas = new MultipleActionServlet(sas, invs); - ReplicationListener replListener1 = getReplicationListener(mgr1.getCache()); + ReplicationListener replListener1 = replListeners[1]; replListener1.expectWithTx(PutDataMapCommand.class, RemoveNodeCommand.class); replListener1.expect(DataGravitationCleanupCommand.class); Modified: core/trunk/src/test/java/org/jboss/cache/integration/websession/util/SessionManager.java =================================================================== --- core/trunk/src/test/java/org/jboss/cache/integration/websession/util/SessionManager.java 2009-02-04 11:19:35 UTC (rev 7636) +++ core/trunk/src/test/java/org/jboss/cache/integration/websession/util/SessionManager.java 2009-02-04 14:19:55 UTC (rev 7637) @@ -35,6 +35,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.jboss.cache.*; +import org.jboss.cache.loader.CacheLoaderManager; import org.jboss.cache.buddyreplication.BuddyManager; import org.jboss.cache.config.BuddyReplicationConfig; import org.jboss.cache.integration.websession.util.WebAppMetadata.Granularity; @@ -131,13 +132,38 @@ cache.removeCacheListener(this); // FIXME see if we need more sophisticated cache cleanup - cache.getInvocationContext().getOptionOverrides().setCacheModeLocal(true); - cache.removeNode(baseFqn); + removeInMemoryData((CacheSPI)cache); + clearCacheLoader((CacheSPI)cache); cacheManager.releaseCache(appMetadata.cacheConfigName); cache = null; } } } + + private void clearCacheLoader(CacheSPI cache) + { + CacheLoaderManager cacheLoaderManager = cache.getCacheLoaderManager(); + if (cacheLoaderManager != null && cacheLoaderManager.getCacheLoader() != null) + { + try + { + cacheLoaderManager.getCacheLoader().remove(Fqn.ROOT); + } catch (Exception e) + { + throw new RuntimeException(e); + } + } + } + + private void removeInMemoryData(CacheSPI cache) + { + if (cache.getRoot() != null) + { + cache.getRoot().clearDataDirect(); + cache.getRoot().removeChildrenDirect(); + } + } + /** * Allows test driver to mock Tomcat background processes' expiration @@ -365,6 +391,7 @@ { cache.getInvocationContext().getOptionOverrides().setCacheModeLocal(true); } + log("cache.removeNode(" + getSessionFqn(id) + ") locally? " + localOnly); cache.removeNode(getSessionFqn(id)); } @@ -424,7 +451,8 @@ { data.put(ATTRIBUTES, attributes); } - + + log("cache.put(" + fqn +"," + data +")"); cache.put(fqn, data); } @@ -438,7 +466,8 @@ Fqn<String> fqn = getSessionFqn(id); if (modifiedAttributes != null) - { + { + log("cache.put(" + fqn+"," + modifiedAttributes+ ")"); cache.put(fqn, modifiedAttributes); } @@ -446,6 +475,7 @@ { for (String key : removedAttributes) { + log("cache.remove(" + fqn + "," + key + ")"); cache.remove(fqn, key); } } @@ -490,6 +520,7 @@ { cache.getInvocationContext().getOptionOverrides().setForceDataGravitation(true); } + log ("cache.getData(" + getSessionFqn(id) + ") with ForceDataGravitation(true)"); data = cache.getData(getSessionFqn(id)); } finally @@ -544,6 +575,7 @@ Fqn<String> backupFqn = Fqn.fromRelativeFqn(child.getFqn(), mainFqn); if (evict) { + log("cache.evict(" + backupFqn + ", true)"); cache.evict(backupFqn, true); } else @@ -556,4 +588,9 @@ } + private void log(String what) + { + System.out.println("[" + cache.getLocalAddress() + "] " + what); + } + } Modified: core/trunk/src/test/java/org/jboss/cache/integration/websession/util/SessionMetadata.java =================================================================== --- core/trunk/src/test/java/org/jboss/cache/integration/websession/util/SessionMetadata.java 2009-02-04 11:19:35 UTC (rev 7636) +++ core/trunk/src/test/java/org/jboss/cache/integration/websession/util/SessionMetadata.java 2009-02-04 14:19:55 UTC (rev 7637) @@ -39,4 +39,15 @@ public final String id = String.valueOf(++counter); public final long creationTime = System.currentTimeMillis(); public boolean valid = true; + + + @Override + public String toString() + { + return "SessionMetadata{" + + "id='" + id + '\'' + + ", creationTime=" + creationTime + + ", valid=" + valid + + '}'; + } } Modified: core/trunk/src/test/java/org/jboss/cache/integration/websession/util/WebSessionTestBase.java =================================================================== --- core/trunk/src/test/java/org/jboss/cache/integration/websession/util/WebSessionTestBase.java 2009-02-04 11:19:35 UTC (rev 7636) +++ core/trunk/src/test/java/org/jboss/cache/integration/websession/util/WebSessionTestBase.java 2009-02-04 14:19:55 UTC (rev 7637) @@ -52,7 +52,7 @@ private AtomicInteger testCount = new AtomicInteger(); private List<CacheManager> cacheManagers; private List<SessionManager> sessionManagers; - private Map<Cache<Object, Object>, ReplicationListener> replicationListeners = new HashMap<Cache<Object, Object>, ReplicationListener>(); + protected ReplicationListener[] replListeners; @BeforeClass(alwaysRun = true) public void beforeClass() throws Exception @@ -61,8 +61,10 @@ if (getStartCachesInBeforeClass() && getCacheConfigName() != null) { String inUseProtocolStack = UnitTestConfigurationFactory.getEmptyConfiguration().getClusterConfig(); - for (CacheManager cm : cacheManagers) + replListeners = new ReplicationListener[getNumCacheManagers()]; + for (int i =0; i < cacheManagers.size(); i++) { + CacheManager cm = cacheManagers.get(i); Cache<Object, Object> cache = cm.getCache(getCacheConfigName(), true); amendCacheBeforeStartup(cache); if (cache.getCacheStatus() != CacheStatus.STARTED) @@ -70,7 +72,7 @@ cache.getConfiguration().setClusterConfig(inUseProtocolStack); cache.start(); } - replicationListeners.put(cache, ReplicationListener.getReplicationListener(cache)); + replListeners[i] = ReplicationListener.getReplicationListener(cache); } } } @@ -171,9 +173,4 @@ { return Integer.valueOf(value); } - - protected ReplicationListener getReplicationListener(Cache<Object, Object> cache) - { - return replicationListeners.get(cache); - } } Modified: core/trunk/src/test/java/org/jboss/cache/util/internals/replicationlisteners/ReplicationListener.java =================================================================== --- core/trunk/src/test/java/org/jboss/cache/util/internals/replicationlisteners/ReplicationListener.java 2009-02-04 11:19:35 UTC (rev 7636) +++ core/trunk/src/test/java/org/jboss/cache/util/internals/replicationlisteners/ReplicationListener.java 2009-02-04 14:19:55 UTC (rev 7637) @@ -159,13 +159,15 @@ Class<? extends ReplicableCommand> replicableCommandClass = it.next(); if (realOne.containsCommandType(replicableCommandClass)) { - it.remove(); + it.remove();//only remove once + break; } else if (realOne.getSingleModification() instanceof PrepareCommand) //explicit transaction { PrepareCommand prepareCommand = (PrepareCommand) realOne.getSingleModification(); if (prepareCommand.containsModificationType(replicableCommandClass)) { it.remove(); + break;//only remove once } } } @@ -254,3 +256,5 @@ return cache; } } +//[127.0.0.1:7900] Processed command: ReplicateCommand{cmds=PrepareCommand{globalTransaction=GlobalTransaction:<127.0.0.1:7903>:481, modifications=[PutDataMapCommand{fqn=/_BUDDY_BACKUP_/127.0.0.1_7903/JSESSION/BuddyReplicationFailoverTest54_localhost/160, dataVersion=null, data={2=org.jboss.cache.integration.websession.util.SessionMetadata@93a0d8, 1=1233588409000, 3={key=2}, 0=3}, globalTransaction=null, erase=false}], localAddress=127.0.0.1:7903, onePhaseCommit=true}} +//[127.0.0.1:7900] skipping command ReplicateCommand{cmds=PrepareCommand{globalTransaction=GlobalTransaction:<127.0.0.1:7903>:482, modifications=[PutDataMapCommand{fqn=/_BUDDY_BACKUP_/127.0.0.1_7903/JSESSION/BuddyReplicationFailoverTest54_localhost/160, dataVersion=null, data={1=1233588409031, 3={key=3}, 0=4}, globalTransaction=null, erase=false}], localAddress=127.0.0.1:7903, onePhaseCommit=true}} Modified: core/trunk/src/test/resources/configs/integration/web-session-cache-configs.xml =================================================================== --- core/trunk/src/test/resources/configs/integration/web-session-cache-configs.xml 2009-02-04 11:19:35 UTC (rev 7636) +++ core/trunk/src/test/resources/configs/integration/web-session-cache-configs.xml 2009-02-04 14:19:55 UTC (rev 7637) @@ -263,7 +263,7 @@ <attribute name="IsolationLevel">REPEATABLE_READ</attribute> <!-- Valid modes are REPL_ASYNC and REPL_SYNC --> - <attribute name="CacheMode">REPL_ASYNC</attribute> + <attribute name="CacheMode">REPL_SYNC</attribute> <!-- Name of cluster. Needs to be the same for all members, in order to find each other --> @@ -341,26 +341,26 @@ </attribute> <!-- Cache Loader for session passivation --> - <!--<attribute name="CacheLoaderConfig">--> - <!--<config>--> - <!--<passivation>true</passivation>--> - <!--<shared>false</shared>--> + <attribute name="CacheLoaderConfig"> + <config> + <passivation>true</passivation> + <shared>false</shared> - <!--<purgeOnStartup>true</purgeOnStartup>--> + <purgeOnStartup>true</purgeOnStartup> - <!--<cacheloader>--> - <!--<class>org.jboss.cache.loader.FileCacheLoader</class>--> - <!--<properties>--> - <!--location=${java.io.tmpdir}${/}session--> - <!--</properties>--> - <!--<async>false</async>--> - <!--<fetchPersistentState>true</fetchPersistentState>--> - <!--<ignoreModifications>false</ignoreModifications>--> - <!--<checkCharacterPortability>false</checkCharacterPortability>--> - <!--</cacheloader>--> + <cacheloader> + <class>org.jboss.cache.loader.FileCacheLoader</class> + <properties> + location=${java.io.tmpdir}${/}session + </properties> + <async>false</async> + <fetchPersistentState>true</fetchPersistentState> + <ignoreModifications>false</ignoreModifications> + <checkCharacterPortability>false</checkCharacterPortability> + </cacheloader> - <!--</config>--> - <!--</attribute>--> + </config> + </attribute> <!-- JBoss Cache eviction is not needed; webapp or SFSB container