[infinispan-issues] [JBoss JIRA] (ISPN-9497) Deadlock when starting multiple servers in parallel

Deadlock when starting multiple servers in parallel --------------------------------------------------- Key: ISPN-9497 URL: https://issues.jboss.org/browse/ISPN-9497 Project: Infinispan Issue Type: Bug Components: Server Affects Versions: 9.4.0.CR2 Reporter: Gustavo Fernandes Priority: Critical Attachments: s1, s2, s3 Observed in the Infinispan Spark test suite, that starts servers in parallel, with separate data folders and different port offsets. One of the server stay blocked when starting the default cache and waiting for the state transfer: {noformat} "MSC service thread 1-2" #15 prio=5 os_prio=0 tid=0x00007f31a4a0e000 nid=0x2c91 waiting on condition [0x00007f313f69d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000f7ab1490> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.infinispan.statetransfer.StateTransferManagerImpl.waitForInitialStateTransferToComplete(StateTransferManagerImpl.java:231) at sun.reflect.GeneratedMethodAccessor74.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:79) at org.infinispan.commons.util.SecurityActions$$Lambda$460/1461359089.run(Unknown Source) at org.infinispan.commons.util.SecurityActions.doPrivileged(SecurityActions.java:71) at org.infinispan.commons.util.SecurityActions.invokeAccessibly(SecurityActions.java:76) at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:185) at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:968) at org.infinispan.factories.AbstractComponentRegistry.lambda$invokePrioritizedMethods$6(AbstractComponentRegistry.java:703) at org.infinispan.factories.AbstractComponentRegistry$$Lambda$474/1392117011.run(Unknown Source) at org.infinispan.factories.SecurityActions.lambda$run$1(SecurityActions.java:72) at org.infinispan.factories.SecurityActions$$Lambda$475/1861961589.run(Unknown Source) at org.infinispan.security.Security.doPrivileged(Security.java:49) at org.infinispan.factories.SecurityActions.run(SecurityActions.java:71) at org.infinispan.factories.AbstractComponentRegistry.invokePrioritizedMethods(AbstractComponentRegistry.java:696) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:689) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:607) at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:244) at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1051) at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:421) at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:646) at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:591) at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:477) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:463) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:496) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:487) at org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:94) at org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:85) at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26) at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14) at org.infinispan.security.Security.doPrivileged(Security.java:49) at org.infinispan.server.hotrod.SecurityActions.doPrivileged(SecurityActions.java:36) at org.infinispan.server.hotrod.SecurityActions.getCache(SecurityActions.java:53) at org.infinispan.server.hotrod.HotRodServer.getCacheInstance(HotRodServer.java:450) at org.infinispan.server.hotrod.HotRodServer.preStartCaches(HotRodServer.java:330) at org.infinispan.server.hotrod.HotRodServer.startTransport(HotRodServer.java:314) at org.infinispan.server.core.AbstractProtocolServer.startInternal(AbstractProtocolServer.java:60) at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:250) at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:106) at org.infinispan.server.core.AbstractProtocolServer.start(AbstractProtocolServer.java:78) at org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:136) at org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:133) at org.infinispan.security.Security.doPrivileged(Security.java:44) at org.infinispan.server.endpoint.subsystem.SecurityActions.doPrivileged(SecurityActions.java:42) at org.infinispan.server.endpoint.subsystem.SecurityActions.startProtocolServer(SecurityActions.java:140) at org.infinispan.server.endpoint.subsystem.ProtocolServerService.startProtocolServer(ProtocolServerService.java:194) at org.infinispan.server.endpoint.subsystem.ProtocolServerService.start(ProtocolServerService.java:152) - locked <0x00000000e11f13f0> (a org.infinispan.server.endpoint.subsystem.ProtocolServerService) at org.jboss.msc.service.ServiceControllerImpl$StartTask.startService(ServiceControllerImpl.java:1736) at org.jboss.msc.service.ServiceControllerImpl$StartTask.execute(ServiceControllerImpl.java:1698) at org.jboss.msc.service.ServiceControllerImpl$ControllerTask.run(ServiceControllerImpl.java:1556) at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35) at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1985) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1487) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1364) at java.lang.Thread.run(Thread.java:748) {noformat} Another server keeps blocked during joining: {noformat} "MSC service thread 1-2" #15 prio=5 os_prio=0 tid=0x00007fec98a02000 nid=0x2f9f waiting on condition [0x00007fec8046c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000f6182fa8> (a java.util.concurrent.CompletableFuture$Signaller) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1695) at java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323) at java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1775) at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1915) at org.infinispan.util.concurrent.CompletableFutures.await(CompletableFutures.java:93) at org.infinispan.remoting.transport.Transport.invokeRemotely(Transport.java:72) at org.infinispan.topology.LocalTopologyManagerImpl.executeOnCoordinator(LocalTopologyManagerImpl.java:688) at org.infinispan.topology.LocalTopologyManagerImpl.join(LocalTopologyManagerImpl.java:148) at org.infinispan.statetransfer.StateTransferManagerImpl.start(StateTransferManagerImpl.java:109) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:79) at org.infinispan.commons.util.SecurityActions$$Lambda$467/848998352.run(Unknown Source) at org.infinispan.commons.util.SecurityActions.doPrivileged(SecurityActions.java:71) at org.infinispan.commons.util.SecurityActions.invokeAccessibly(SecurityActions.java:76) at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:185) at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:968) at org.infinispan.factories.AbstractComponentRegistry.lambda$invokePrioritizedMethods$6(AbstractComponentRegistry.java:703) at org.infinispan.factories.AbstractComponentRegistry$$Lambda$481/252402945.run(Unknown Source) at org.infinispan.factories.SecurityActions.lambda$run$1(SecurityActions.java:72) at org.infinispan.factories.SecurityActions$$Lambda$482/1109883309.run(Unknown Source) at org.infinispan.security.Security.doPrivileged(Security.java:49) at org.infinispan.factories.SecurityActions.run(SecurityActions.java:71) at org.infinispan.factories.AbstractComponentRegistry.invokePrioritizedMethods(AbstractComponentRegistry.java:696) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:689) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:607) at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:244) at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1051) at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:421) at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:646) at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:591) at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:477) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:463) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:496) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:487) at org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:94) at org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:85) at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26) at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14) at org.infinispan.security.Security.doPrivileged(Security.java:49) at org.infinispan.server.hotrod.SecurityActions.doPrivileged(SecurityActions.java:36) at org.infinispan.server.hotrod.SecurityActions.getCache(SecurityActions.java:53) at org.infinispan.server.hotrod.HotRodServer.getCacheInstance(HotRodServer.java:450) at org.infinispan.server.hotrod.HotRodServer.preStartCaches(HotRodServer.java:330) at org.infinispan.server.hotrod.HotRodServer.startTransport(HotRodServer.java:314) at org.infinispan.server.core.AbstractProtocolServer.startInternal(AbstractProtocolServer.java:60) at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:250) at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:106) at org.infinispan.server.core.AbstractProtocolServer.start(AbstractProtocolServer.java:78) at org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:136) at org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:133) at org.infinispan.security.Security.doPrivileged(Security.java:44) at org.infinispan.server.endpoint.subsystem.SecurityActions.doPrivileged(SecurityActions.java:42) at org.infinispan.server.endpoint.subsystem.SecurityActions.startProtocolServer(SecurityActions.java:140) at org.infinispan.server.endpoint.subsystem.ProtocolServerService.startProtocolServer(ProtocolServerService.java:194) at org.infinispan.server.endpoint.subsystem.ProtocolServerService.start(ProtocolServerService.java:152) - locked <0x00000000e0f59e90> (a org.infinispan.server.endpoint.subsystem.ProtocolServerService) at org.jboss.msc.service.ServiceControllerImpl$StartTask.startService(ServiceControllerImpl.java:1736) at org.jboss.msc.service.ServiceControllerImpl$StartTask.execute(ServiceControllerImpl.java:1698) at org.jboss.msc.service.ServiceControllerImpl$ControllerTask.run(ServiceControllerImpl.java:1556) at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35) at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1985) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1487) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1378) at java.lang.Thread.run(Thread.java:748) {noformat}