InitialClusterSizeTest can hang during teardown ----------------------------------------------- Key: ISPN-7439 URL: https://issues.jboss.org/browse/ISPN-7439 Project: Infinispan Issue Type: Bug Components: Core, Test Suite - Core Affects Versions: 9.0.0.Beta2 Reporter: Dan Berindei Assignee: Pedro Ruivo Labels: testsuite_stability Test {{testInitialClusterSizeFail}} expects the nodes to time out in {{JGroupsTransport.waitForInitialNodes()}}, but in at least one case the timeout didn't happen. The test then tried to shut down the cache managers, but it hanged because another thread was holding the {{GlobalComponentRegistry}} lock: {noformat} "testng-InitialClusterSizeTest" #13 prio=5 os_prio=0 tid=0x00007f1874d1f000 nid=0x1778 waiting for monitor entry [0x00007f181bafc000] java.lang.Thread.State: BLOCKED (on object monitor) at org.infinispan.factories.GlobalComponentRegistry.stop(GlobalComponentRegistry.java:280) - waiting to lock <0x0000000093c7afe0> (a org.infinispan.factories.GlobalComponentRegistry) at org.infinispan.manager.DefaultCacheManager.stop(DefaultCacheManager.java:701) - locked <0x000000008a005b80> (a org.infinispan.manager.DefaultCacheManager) at org.infinispan.test.TestingUtil.killCacheManagers(TestingUtil.java:656) at org.infinispan.test.MultipleCacheManagersTest.clearContent(MultipleCacheManagersTest.java:138) at sun.reflect.GeneratedMethodAccessor175.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84) at org.testng.internal.Invoker.invokeConfigurationMethod(Invoker.java:564) at org.testng.internal.Invoker.invokeConfigurations(Invoker.java:213) at org.testng.internal.Invoker.invokeMethod(Invoker.java:786) at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901) at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231) at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127) at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111) at org.testng.TestRunner.privateRun(TestRunner.java:767) at org.testng.TestRunner.run(TestRunner.java:617) at org.testng.SuiteRunner.runTest(SuiteRunner.java:348) at org.testng.SuiteRunner.access$000(SuiteRunner.java:38) at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:382) at org.testng.internal.thread.ThreadUtil$2.call(ThreadUtil.java:64) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ForkThread-4,InitialClusterSizeTest" #167842 prio=5 os_prio=0 tid=0x00007f1824163800 nid=0x3316 waiting on condition [0x00007f17e62b9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.jgroups.util.Util.sleep(Util.java:1818) at org.jgroups.protocols.pbcast.ClientGmsImpl.firstOfAllClients(ClientGmsImpl.java:181) at org.jgroups.protocols.pbcast.ClientGmsImpl.joinInternal(ClientGmsImpl.java:97) at org.jgroups.protocols.pbcast.ClientGmsImpl.join(ClientGmsImpl.java:41) at org.jgroups.protocols.pbcast.GMS.down(GMS.java:1066) at org.jgroups.protocols.tom.TOA.down(TOA.java:73) at org.jgroups.protocols.FlowControl.down(FlowControl.java:302) at org.jgroups.protocols.RSVP.down(RSVP.java:102) at org.jgroups.stack.ProtocolStack.down(ProtocolStack.java:900) at org.jgroups.JChannel.down(JChannel.java:644) at org.jgroups.JChannel._connect(JChannel.java:873) at org.jgroups.JChannel.connect(JChannel.java:369) - locked <0x0000000093c7aea0> (a org.jgroups.JChannel) at org.jgroups.JChannel.connect(JChannel.java:360) - locked <0x0000000093c7aea0> (a org.jgroups.JChannel) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.startJGroupsChannelIfNeeded(JGroupsTransport.java:221) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.start(JGroupsTransport.java:211) at sun.reflect.GeneratedMethodAccessor109.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:168) at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:867) at org.infinispan.factories.AbstractComponentRegistry.invokeStartMethods(AbstractComponentRegistry.java:633) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:622) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:547) - locked <0x0000000093c7afe0> (a org.infinispan.factories.GlobalComponentRegistry) at org.infinispan.factories.GlobalComponentRegistry.start(GlobalComponentRegistry.java:244) - locked <0x0000000093c7afe0> (a org.infinispan.factories.GlobalComponentRegistry) at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:666) at org.infinispan.remoting.transport.InitialClusterSizeTest.lambda$testInitialClusterSize$799(InitialClusterSizeTest.java:47) at org.infinispan.remoting.transport.InitialClusterSizeTest$$Lambda$2092/593962598.run(Unknown Source) at org.infinispan.test.AbstractInfinispanTest$RunnableWrapper.run(AbstractInfinispanTest.java:510) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ForkThread-2,InitialClusterSizeTest" #167840 prio=5 os_prio=0 tid=0x00007f1824164800 nid=0x3314 waiting on condition [0x00007f17eec40000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.jgroups.util.Util.sleep(Util.java:1818) at org.jgroups.protocols.pbcast.ClientGmsImpl.firstOfAllClients(ClientGmsImpl.java:181) at org.jgroups.protocols.pbcast.ClientGmsImpl.joinInternal(ClientGmsImpl.java:97) at org.jgroups.protocols.pbcast.ClientGmsImpl.join(ClientGmsImpl.java:41) at org.jgroups.protocols.pbcast.GMS.down(GMS.java:1066) at org.jgroups.protocols.tom.TOA.down(TOA.java:73) at org.jgroups.protocols.FlowControl.down(FlowControl.java:302) at org.jgroups.protocols.RSVP.down(RSVP.java:102) at org.jgroups.stack.ProtocolStack.down(ProtocolStack.java:900) at org.jgroups.JChannel.down(JChannel.java:644) at org.jgroups.JChannel._connect(JChannel.java:873) at org.jgroups.JChannel.connect(JChannel.java:369) - locked <0x0000000093c7b7f0> (a org.jgroups.JChannel) at org.jgroups.JChannel.connect(JChannel.java:360) - locked <0x0000000093c7b7f0> (a org.jgroups.JChannel) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.startJGroupsChannelIfNeeded(JGroupsTransport.java:221) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.start(JGroupsTransport.java:211) at sun.reflect.GeneratedMethodAccessor109.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:168) at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:867) at org.infinispan.factories.AbstractComponentRegistry.invokeStartMethods(AbstractComponentRegistry.java:633) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:622) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:547) - locked <0x0000000093c7b930> (a org.infinispan.factories.GlobalComponentRegistry) at org.infinispan.factories.GlobalComponentRegistry.start(GlobalComponentRegistry.java:244) - locked <0x0000000093c7b930> (a org.infinispan.factories.GlobalComponentRegistry) at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:666) at org.infinispan.remoting.transport.InitialClusterSizeTest.lambda$testInitialClusterSize$799(InitialClusterSizeTest.java:47) at org.infinispan.remoting.transport.InitialClusterSizeTest$$Lambda$2092/593962598.run(Unknown Source) at org.infinispan.test.AbstractInfinispanTest$RunnableWrapper.run(AbstractInfinispanTest.java:510) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) {noformat} See the full thread dump here: http://ci.infinispan.org/viewLog.html?buildId=49393&buildTypeId=bt9