Author: jmesnil Date: 2009-12-09 14:00:06 -0500 (Wed, 09 Dec 2009) New Revision: 8648 Modified: trunk/tests/src/org/hornetq/tests/integration/cluster/failover/ClusterWithBackupFailoverTestBase.java Log: failover tests for symmetric cluster * added test when failing backup nodes * added test to fail both live and activated backup nodes Modified: trunk/tests/src/org/hornetq/tests/integration/cluster/failover/ClusterWithBackupFailoverTestBase.java =================================================================== --- trunk/tests/src/org/hornetq/tests/integration/cluster/failover/ClusterWithBackupFailoverTestBase.java 2009-12-09 18:59:58 UTC (rev 8647) +++ trunk/tests/src/org/hornetq/tests/integration/cluster/failover/ClusterWithBackupFailoverTestBase.java 2009-12-09 19:00:06 UTC (rev 8648) @@ -75,7 +75,7 @@ return false; } - public void testFailAllNodes() throws Exception + public void testFailLiveNodes() throws Exception { setupCluster(); @@ -187,7 +187,110 @@ ClusterWithBackupFailoverTestBase.log.info("*** test done"); } + + public void testFailBackupNodes() throws Exception + { + setupCluster(); + startServers(3, 4, 5, 0, 1, 2); + + setupSessionFactory(0, 3, isNetty(), false); + setupSessionFactory(1, 4, isNetty(), false); + setupSessionFactory(2, 5, isNetty(), false); + + createQueue(0, "queues.testaddress", "queue0", null, true); + createQueue(1, "queues.testaddress", "queue0", null, true); + createQueue(2, "queues.testaddress", "queue0", null, true); + + addConsumer(0, 0, "queue0", null); + addConsumer(1, 1, "queue0", null); + addConsumer(2, 2, "queue0", null); + + waitForBindings(0, "queues.testaddress", 1, 1, true); + waitForBindings(1, "queues.testaddress", 1, 1, true); + waitForBindings(2, "queues.testaddress", 1, 1, true); + + waitForBindings(0, "queues.testaddress", 2, 2, false); + waitForBindings(1, "queues.testaddress", 2, 2, false); + waitForBindings(2, "queues.testaddress", 2, 2, false); + + send(0, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(1, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(2, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + failNode(3); + + waitForBindings(0, "queues.testaddress", 1, 1, true); + waitForBindings(1, "queues.testaddress", 1, 1, true); + waitForBindings(2, "queues.testaddress", 1, 1, true); + + waitForBindings(0, "queues.testaddress", 2, 2, false); + waitForBindings(1, "queues.testaddress", 2, 2, false); + waitForBindings(2, "queues.testaddress", 2, 2, false); + + send(0, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(1, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(2, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + failNode(4); + + waitForBindings(0, "queues.testaddress", 1, 1, true); + waitForBindings(1, "queues.testaddress", 1, 1, true); + waitForBindings(2, "queues.testaddress", 1, 1, true); + + waitForBindings(0, "queues.testaddress", 2, 2, false); + waitForBindings(1, "queues.testaddress", 2, 2, false); + waitForBindings(2, "queues.testaddress", 2, 2, false); + + send(0, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(1, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(2, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + failNode(5); + + waitForBindings(0, "queues.testaddress", 1, 1, true); + waitForBindings(1, "queues.testaddress", 1, 1, true); + waitForBindings(2, "queues.testaddress", 1, 1, true); + + waitForBindings(0, "queues.testaddress", 2, 2, false); + waitForBindings(1, "queues.testaddress", 2, 2, false); + waitForBindings(2, "queues.testaddress", 2, 2, false); + + send(0, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(1, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(2, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + + + removeConsumer(0); + removeConsumer(1); + removeConsumer(2); + + stopServers(); + + ClusterWithBackupFailoverTestBase.log.info("*** test done"); + } + protected void setupCluster() throws Exception { setupCluster(false); @@ -224,15 +327,141 @@ // Prevent remoting service taking any more connections server.getRemotingService().freeze(); - // Stop it broadcasting - for (BroadcastGroup group : server.getClusterManager().getBroadcastGroups()) + if (server.getClusterManager() != null) { - group.stop(); + // Stop it broadcasting + for (BroadcastGroup group : server.getClusterManager().getBroadcastGroups()) + { + group.stop(); + } } - + FailoverManagerImpl.failAllConnectionsForConnector(serverTC); server.stop(); } + public void testFailAllNodes() throws Exception + { + setupCluster(); + + startServers(3, 4, 5, 0, 1, 2); + + setupSessionFactory(0, 3, isNetty(), false); + setupSessionFactory(1, 4, isNetty(), false); + setupSessionFactory(2, 5, isNetty(), false); + + createQueue(0, "queues.testaddress", "queue0", null, true); + createQueue(1, "queues.testaddress", "queue0", null, true); + createQueue(2, "queues.testaddress", "queue0", null, true); + + addConsumer(0, 0, "queue0", null); + addConsumer(1, 1, "queue0", null); + addConsumer(2, 2, "queue0", null); + + waitForBindings(0, "queues.testaddress", 1, 1, true); + waitForBindings(1, "queues.testaddress", 1, 1, true); + waitForBindings(2, "queues.testaddress", 1, 1, true); + + waitForBindings(0, "queues.testaddress", 2, 2, false); + waitForBindings(1, "queues.testaddress", 2, 2, false); + waitForBindings(2, "queues.testaddress", 2, 2, false); + + send(0, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(1, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(2, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + failNode(0); + + // live nodes + waitForBindings(1, "queues.testaddress", 1, 1, true); + waitForBindings(2, "queues.testaddress", 1, 1, true); + // activated backup nodes + waitForBindings(3, "queues.testaddress", 1, 1, true); + + // live nodes + waitForBindings(1, "queues.testaddress", 2, 2, false); + waitForBindings(2, "queues.testaddress", 2, 2, false); + // activated backup nodes + waitForBindings(3, "queues.testaddress", 2, 2, false); + + ClusterWithBackupFailoverTestBase.log.info("** now sending"); + + send(0, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(1, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + send(2, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 0, 1, 2); + + removeConsumer(0); + failNode(3); + + // live nodes + waitForBindings(1, "queues.testaddress", 1, 1, true); + waitForBindings(2, "queues.testaddress", 1, 1, true); + + // live nodes + waitForBindings(1, "queues.testaddress", 1, 1, false); + waitForBindings(2, "queues.testaddress", 1, 1, false); + + send(1, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 1, 2); + + send(2, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 1, 2); + + failNode(1); + + // live nodes + waitForBindings(2, "queues.testaddress", 1, 1, true); + // activated backup nodes + waitForBindings(4, "queues.testaddress", 1, 1, true); + + // live nodes + waitForBindings(2, "queues.testaddress", 1, 1, false); + // activated backup nodes + waitForBindings(4, "queues.testaddress", 1, 1, false); + + send(1, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 1, 2); + + send(2, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 1, 2); + + removeConsumer(1); + failNode(4); + + // live nodes + waitForBindings(2, "queues.testaddress", 1, 1, true); + // live nodes + waitForBindings(2, "queues.testaddress", 1, 0, false); + + send(2, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 2); + + failNode(2); + + // live nodes + waitForBindings(5, "queues.testaddress", 1, 1, true); + // live nodes + waitForBindings(5, "queues.testaddress", 0, 0, false); + + send(2, "queues.testaddress", 10, false, null); + verifyReceiveRoundRobinInSomeOrder(true, 10, 2); + + removeConsumer(2); + failNode(5); + + stopServers(); + + ClusterWithBackupFailoverTestBase.log.info("*** test done"); + } }