[infinispan-issues] [JBoss JIRA] (ISPN-9497) Deadlock when starting multiple servers in parallel
Gustavo Fernandes (JIRA)
issues at jboss.org
Mon Sep 10 04:31:08 EDT 2018
[ https://issues.jboss.org/browse/ISPN-9497?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Gustavo Fernandes updated ISPN-9497:
------------------------------------
Attachment: s3
s2
s1
> Deadlock when starting multiple servers in parallel
> ---------------------------------------------------
>
> Key: ISPN-9497
> URL: https://issues.jboss.org/browse/ISPN-9497
> Project: Infinispan
> Issue Type: Bug
> Components: Server
> Affects Versions: 9.4.0.CR2
> Reporter: Gustavo Fernandes
> Priority: Critical
> Attachments: s1, s2, s3
>
>
> Observed in the Infinispan Spark test suite, that starts servers in parallel, with separate data folders and different port offsets.
> One of the server stay blocked when starting the default cache and waiting for the state transfer:
> {noformat}
> "MSC service thread 1-2" #15 prio=5 os_prio=0 tid=0x00007f31a4a0e000 nid=0x2c91 waiting on condition [0x00007f313f69d000]
> java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x00000000f7ab1490> (a java.util.concurrent.CountDownLatch$Sync)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
> at org.infinispan.statetransfer.StateTransferManagerImpl.waitForInitialStateTransferToComplete(StateTransferManagerImpl.java:231)
> at sun.reflect.GeneratedMethodAccessor74.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:79)
> at org.infinispan.commons.util.SecurityActions$$Lambda$460/1461359089.run(Unknown Source)
> at org.infinispan.commons.util.SecurityActions.doPrivileged(SecurityActions.java:71)
> at org.infinispan.commons.util.SecurityActions.invokeAccessibly(SecurityActions.java:76)
> at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:185)
> at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:968)
> at org.infinispan.factories.AbstractComponentRegistry.lambda$invokePrioritizedMethods$6(AbstractComponentRegistry.java:703)
> at org.infinispan.factories.AbstractComponentRegistry$$Lambda$474/1392117011.run(Unknown Source)
> at org.infinispan.factories.SecurityActions.lambda$run$1(SecurityActions.java:72)
> at org.infinispan.factories.SecurityActions$$Lambda$475/1861961589.run(Unknown Source)
> at org.infinispan.security.Security.doPrivileged(Security.java:49)
> at org.infinispan.factories.SecurityActions.run(SecurityActions.java:71)
> at org.infinispan.factories.AbstractComponentRegistry.invokePrioritizedMethods(AbstractComponentRegistry.java:696)
> at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:689)
> at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:607)
> at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:244)
> at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1051)
> at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:421)
> at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:646)
> at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:591)
> at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:477)
> at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:463)
> at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:496)
> at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:487)
> at org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:94)
> at org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:85)
> at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26)
> at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14)
> at org.infinispan.security.Security.doPrivileged(Security.java:49)
> at org.infinispan.server.hotrod.SecurityActions.doPrivileged(SecurityActions.java:36)
> at org.infinispan.server.hotrod.SecurityActions.getCache(SecurityActions.java:53)
> at org.infinispan.server.hotrod.HotRodServer.getCacheInstance(HotRodServer.java:450)
> at org.infinispan.server.hotrod.HotRodServer.preStartCaches(HotRodServer.java:330)
> at org.infinispan.server.hotrod.HotRodServer.startTransport(HotRodServer.java:314)
> at org.infinispan.server.core.AbstractProtocolServer.startInternal(AbstractProtocolServer.java:60)
> at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:250)
> at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:106)
> at org.infinispan.server.core.AbstractProtocolServer.start(AbstractProtocolServer.java:78)
> at org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:136)
> at org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:133)
> at org.infinispan.security.Security.doPrivileged(Security.java:44)
> at org.infinispan.server.endpoint.subsystem.SecurityActions.doPrivileged(SecurityActions.java:42)
> at org.infinispan.server.endpoint.subsystem.SecurityActions.startProtocolServer(SecurityActions.java:140)
> at org.infinispan.server.endpoint.subsystem.ProtocolServerService.startProtocolServer(ProtocolServerService.java:194)
> at org.infinispan.server.endpoint.subsystem.ProtocolServerService.start(ProtocolServerService.java:152)
> - locked <0x00000000e11f13f0> (a org.infinispan.server.endpoint.subsystem.ProtocolServerService)
> at org.jboss.msc.service.ServiceControllerImpl$StartTask.startService(ServiceControllerImpl.java:1736)
> at org.jboss.msc.service.ServiceControllerImpl$StartTask.execute(ServiceControllerImpl.java:1698)
> at org.jboss.msc.service.ServiceControllerImpl$ControllerTask.run(ServiceControllerImpl.java:1556)
> at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
> at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1985)
> at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1487)
> at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1364)
> at java.lang.Thread.run(Thread.java:748)
> {noformat}
> Another server keeps blocked during joining:
> {noformat}
> "MSC service thread 1-2" #15 prio=5 os_prio=0 tid=0x00007fec98a02000 nid=0x2f9f waiting on condition [0x00007fec8046c000]
> java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x00000000f6182fa8> (a java.util.concurrent.CompletableFuture$Signaller)
> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1695)
> at java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323)
> at java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1775)
> at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1915)
> at org.infinispan.util.concurrent.CompletableFutures.await(CompletableFutures.java:93)
> at org.infinispan.remoting.transport.Transport.invokeRemotely(Transport.java:72)
> at org.infinispan.topology.LocalTopologyManagerImpl.executeOnCoordinator(LocalTopologyManagerImpl.java:688)
> at org.infinispan.topology.LocalTopologyManagerImpl.join(LocalTopologyManagerImpl.java:148)
> at org.infinispan.statetransfer.StateTransferManagerImpl.start(StateTransferManagerImpl.java:109)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:79)
> at org.infinispan.commons.util.SecurityActions$$Lambda$467/848998352.run(Unknown Source)
> at org.infinispan.commons.util.SecurityActions.doPrivileged(SecurityActions.java:71)
> at org.infinispan.commons.util.SecurityActions.invokeAccessibly(SecurityActions.java:76)
> at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:185)
> at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:968)
> at org.infinispan.factories.AbstractComponentRegistry.lambda$invokePrioritizedMethods$6(AbstractComponentRegistry.java:703)
> at org.infinispan.factories.AbstractComponentRegistry$$Lambda$481/252402945.run(Unknown Source)
> at org.infinispan.factories.SecurityActions.lambda$run$1(SecurityActions.java:72)
> at org.infinispan.factories.SecurityActions$$Lambda$482/1109883309.run(Unknown Source)
> at org.infinispan.security.Security.doPrivileged(Security.java:49)
> at org.infinispan.factories.SecurityActions.run(SecurityActions.java:71)
> at org.infinispan.factories.AbstractComponentRegistry.invokePrioritizedMethods(AbstractComponentRegistry.java:696)
> at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:689)
> at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:607)
> at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:244)
> at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1051)
> at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:421)
> at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:646)
> at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:591)
> at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:477)
> at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:463)
> at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:496)
> at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:487)
> at org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:94)
> at org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:85)
> at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26)
> at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14)
> at org.infinispan.security.Security.doPrivileged(Security.java:49)
> at org.infinispan.server.hotrod.SecurityActions.doPrivileged(SecurityActions.java:36)
> at org.infinispan.server.hotrod.SecurityActions.getCache(SecurityActions.java:53)
> at org.infinispan.server.hotrod.HotRodServer.getCacheInstance(HotRodServer.java:450)
> at org.infinispan.server.hotrod.HotRodServer.preStartCaches(HotRodServer.java:330)
> at org.infinispan.server.hotrod.HotRodServer.startTransport(HotRodServer.java:314)
> at org.infinispan.server.core.AbstractProtocolServer.startInternal(AbstractProtocolServer.java:60)
> at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:250)
> at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:106)
> at org.infinispan.server.core.AbstractProtocolServer.start(AbstractProtocolServer.java:78)
> at org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:136)
> at org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:133)
> at org.infinispan.security.Security.doPrivileged(Security.java:44)
> at org.infinispan.server.endpoint.subsystem.SecurityActions.doPrivileged(SecurityActions.java:42)
> at org.infinispan.server.endpoint.subsystem.SecurityActions.startProtocolServer(SecurityActions.java:140)
> at org.infinispan.server.endpoint.subsystem.ProtocolServerService.startProtocolServer(ProtocolServerService.java:194)
> at org.infinispan.server.endpoint.subsystem.ProtocolServerService.start(ProtocolServerService.java:152)
> - locked <0x00000000e0f59e90> (a org.infinispan.server.endpoint.subsystem.ProtocolServerService)
> at org.jboss.msc.service.ServiceControllerImpl$StartTask.startService(ServiceControllerImpl.java:1736)
> at org.jboss.msc.service.ServiceControllerImpl$StartTask.execute(ServiceControllerImpl.java:1698)
> at org.jboss.msc.service.ServiceControllerImpl$ControllerTask.run(ServiceControllerImpl.java:1556)
> at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
> at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1985)
> at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1487)
> at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1378)
> at java.lang.Thread.run(Thread.java:748)
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.5.0#75005)
More information about the infinispan-issues
mailing list