6:27 a.m.
[
https://issues.jboss.org/browse/ISPN-9497?page=com.atlassian.jira.plugin....
]
Gustavo Fernandes commented on ISPN-9497:
-----------------------------------------
One of the servers failed to start properly due to a timeout during the counter
initialization:
{noformat}
2018-09-10 12:21:38,034 ERROR [org.jboss.msc.service.fail] (MSC service thread 1-4)
MSC000001: Failed to start service jboss.datagrid-infinispan.clustered:
org.jboss.msc.service.StartException in service jboss.datagrid-infinispan.clustered:
Failed to start service
at
org.jboss.msc.service.ServiceControllerImpl$StartTask.execute(ServiceControllerImpl.java:1728)
at
org.jboss.msc.service.ServiceControllerImpl$ControllerTask.run(ServiceControllerImpl.java:1556)
at
org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1985)
at
org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1487)
at
org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1378)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.infinispan.manager.EmbeddedCacheManagerStartupException:
org.infinispan.commons.CacheException: Unable to invoke method public void
org.infinispan.counter.impl.manager.EmbeddedCounterManager.startCaches() on object of type
EmbeddedCounterManager
at
org.infinispan.factories.GlobalComponentRegistry.start(GlobalComponentRegistry.java:271)
at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:680)
at
org.infinispan.manager.impl.AbstractDelegatingEmbeddedCacheManager.start(AbstractDelegatingEmbeddedCacheManager.java:192)
at
org.infinispan.server.infinispan.SecurityActions.lambda$registerAndStartContainer$0(SecurityActions.java:84)
at org.infinispan.security.Security.doPrivileged(Security.java:44)
at
org.infinispan.server.infinispan.SecurityActions.doPrivileged(SecurityActions.java:69)
at
org.infinispan.server.infinispan.SecurityActions.registerAndStartContainer(SecurityActions.java:87)
at
org.jboss.as.clustering.infinispan.subsystem.CacheContainerBuilder.start(CacheContainerBuilder.java:100)
at
org.jboss.msc.service.ServiceControllerImpl$StartTask.startService(ServiceControllerImpl.java:1736)
at
org.jboss.msc.service.ServiceControllerImpl$StartTask.execute(ServiceControllerImpl.java:1698)
... 6 more
Caused by: org.infinispan.commons.CacheException: Unable to invoke method public void
org.infinispan.counter.impl.manager.EmbeddedCounterManager.startCaches() on object of type
EmbeddedCounterManager
at
org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:83)
at org.infinispan.commons.util.SecurityActions.doPrivileged(SecurityActions.java:71)
at org.infinispan.commons.util.SecurityActions.invokeAccessibly(SecurityActions.java:76)
at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:185)
at
org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:968)
at
org.infinispan.factories.AbstractComponentRegistry.lambda$invokePrioritizedMethods$6(AbstractComponentRegistry.java:703)
at org.infinispan.factories.SecurityActions.lambda$run$1(SecurityActions.java:72)
at org.infinispan.security.Security.doPrivileged(Security.java:49)
at org.infinispan.factories.SecurityActions.run(SecurityActions.java:71)
at
org.infinispan.factories.AbstractComponentRegistry.invokePrioritizedMethods(AbstractComponentRegistry.java:696)
at
org.infinispan.factories.AbstractComponentRegistry.postStart(AbstractComponentRegistry.java:620)
at
org.infinispan.factories.GlobalComponentRegistry.start(GlobalComponentRegistry.java:267)
... 15 more
Caused by: org.infinispan.commons.CacheException: java.util.concurrent.TimeoutException
at
org.infinispan.stream.impl.AbstractCacheStream.performOperationRehashAware(AbstractCacheStream.java:332)
at
org.infinispan.stream.impl.AbstractCacheStream.performOperation(AbstractCacheStream.java:229)
at
org.infinispan.stream.impl.DistributedCacheStream.anyMatch(DistributedCacheStream.java:397)
at
org.infinispan.util.AbstractDelegatingCacheStream.anyMatch(AbstractDelegatingCacheStream.java:300)
at org.infinispan.CacheStream.anyMatch(CacheStream.java:462)
at
org.infinispan.counter.impl.manager.CounterConfigurationManager.start(CounterConfigurationManager.java:90)
at
org.infinispan.counter.impl.manager.EmbeddedCounterManager.startCaches(EmbeddedCounterManager.java:97)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:79)
... 26 more
Caused by: java.util.concurrent.TimeoutException
at java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1771)
at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1915)
at
org.infinispan.stream.impl.AbstractCacheStream.performOperationRehashAware(AbstractCacheStream.java:330)
... 37 more
2018-09-10 12:21:38,039 ERROR [org.jboss.as.controller.management-operation] (Controller
Boot Thread) WFLYCTL0013: Operation ("add") failed - address: ([
("subsystem" => "datagrid-infinispan"),
("cache-container" => "clustered")
]) - failure description: {"WFLYCTL0080: Failed services" =>
{"jboss.datagrid-infinispan.clustered" => "Failed to start service
Caused by: org.infinispan.manager.EmbeddedCacheManagerStartupException:
org.infinispan.commons.CacheException: Unable to invoke method public void
org.infinispan.counter.impl.manager.EmbeddedCounterManager.startCaches() on object of type
EmbeddedCounterManager
Caused by: org.infinispan.commons.CacheException: Unable to invoke method public void
org.infinispan.counter.impl.manager.EmbeddedCounterManager.startCaches() on object of type
EmbeddedCounterManager
Caused by: org.infinispan.commons.CacheException:
java.util.concurrent.TimeoutException
Caused by: java.util.concurrent.TimeoutException"}}
2018-09-10 12:21:38,046 INFO [org.jboss.as.controller] (Controller Boot Thread)
WFLYCTL0183: Service status report
WFLYCTL0186: Services which failed to start: service
jboss.datagrid-infinispan.clustered: Failed to start service
WFLYCTL0448: 3 additional services are down due to their dependencies being missing or
failed
2018-09-10 12:21:38,079 INFO [org.jboss.as.server] (Controller Boot Thread) WFLYSRV0212:
Resuming server
2018-09-10 12:21:38,082 INFO [org.jboss.as] (Controller Boot Thread) WFLYSRV0060: Http
management interface listening on http://127.0.0.1:11990/management
2018-09-10 12:21:38,082 INFO [org.jboss.as] (Controller Boot Thread) WFLYSRV0051: Admin
console listening on http://127.0.0.1:11990
2018-09-10 12:21:38,082 ERROR [org.jboss.as] (Controller Boot Thread) WFLYSRV0026:
Infinispan Server 9.4.0-SNAPSHOT (WildFly Core 5.0.0.Final) started (with errors) in
39356ms - Started 208 of 275 services (5 services failed or missing dependencies, 133
services are lazy, passive or on-demand)
{noformat}
[~pruivo] Have you seen ^ before?
Deadlock when starting multiple servers in parallel
---------------------------------------------------
Key: ISPN-9497
URL: https://issues.jboss.org/browse/ISPN-9497
Project: Infinispan
Issue Type: Bug
Components: Server
Affects Versions: 9.4.0.CR2
Reporter: Gustavo Fernandes
Priority: Critical
Attachments: s1, s2, s3
Observed in the Infinispan Spark test suite, that starts servers in parallel, with
separate data folders and different port offsets.
One of the server stay blocked when starting the default cache and waiting for the state
transfer:
{noformat}
"MSC service thread 1-2" #15 prio=5 os_prio=0 tid=0x00007f31a4a0e000 nid=0x2c91
waiting on condition [0x00007f313f69d000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000f7ab1490> (a
java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at
org.infinispan.statetransfer.StateTransferManagerImpl.waitForInitialStateTransferToComplete(StateTransferManagerImpl.java:231)
at sun.reflect.GeneratedMethodAccessor74.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:79)
at org.infinispan.commons.util.SecurityActions$$Lambda$460/1461359089.run(Unknown
Source)
at org.infinispan.commons.util.SecurityActions.doPrivileged(SecurityActions.java:71)
at
org.infinispan.commons.util.SecurityActions.invokeAccessibly(SecurityActions.java:76)
at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:185)
at
org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:968)
at
org.infinispan.factories.AbstractComponentRegistry.lambda$invokePrioritizedMethods$6(AbstractComponentRegistry.java:703)
at org.infinispan.factories.AbstractComponentRegistry$$Lambda$474/1392117011.run(Unknown
Source)
at org.infinispan.factories.SecurityActions.lambda$run$1(SecurityActions.java:72)
at org.infinispan.factories.SecurityActions$$Lambda$475/1861961589.run(Unknown Source)
at org.infinispan.security.Security.doPrivileged(Security.java:49)
at org.infinispan.factories.SecurityActions.run(SecurityActions.java:71)
at
org.infinispan.factories.AbstractComponentRegistry.invokePrioritizedMethods(AbstractComponentRegistry.java:696)
at
org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:689)
at
org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:607)
at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:244)
at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1051)
at
org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:421)
at
org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:646)
at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:591)
at
org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:477)
at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:463)
at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:496)
at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:487)
at
org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:94)
at
org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:85)
at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26)
at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14)
at org.infinispan.security.Security.doPrivileged(Security.java:49)
at org.infinispan.server.hotrod.SecurityActions.doPrivileged(SecurityActions.java:36)
at org.infinispan.server.hotrod.SecurityActions.getCache(SecurityActions.java:53)
at org.infinispan.server.hotrod.HotRodServer.getCacheInstance(HotRodServer.java:450)
at org.infinispan.server.hotrod.HotRodServer.preStartCaches(HotRodServer.java:330)
at org.infinispan.server.hotrod.HotRodServer.startTransport(HotRodServer.java:314)
at
org.infinispan.server.core.AbstractProtocolServer.startInternal(AbstractProtocolServer.java:60)
at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:250)
at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:106)
at
org.infinispan.server.core.AbstractProtocolServer.start(AbstractProtocolServer.java:78)
at
org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:136)
at
org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:133)
at org.infinispan.security.Security.doPrivileged(Security.java:44)
at
org.infinispan.server.endpoint.subsystem.SecurityActions.doPrivileged(SecurityActions.java:42)
at
org.infinispan.server.endpoint.subsystem.SecurityActions.startProtocolServer(SecurityActions.java:140)
at
org.infinispan.server.endpoint.subsystem.ProtocolServerService.startProtocolServer(ProtocolServerService.java:194)
at
org.infinispan.server.endpoint.subsystem.ProtocolServerService.start(ProtocolServerService.java:152)
- locked <0x00000000e11f13f0> (a
org.infinispan.server.endpoint.subsystem.ProtocolServerService)
at
org.jboss.msc.service.ServiceControllerImpl$StartTask.startService(ServiceControllerImpl.java:1736)
at
org.jboss.msc.service.ServiceControllerImpl$StartTask.execute(ServiceControllerImpl.java:1698)
at
org.jboss.msc.service.ServiceControllerImpl$ControllerTask.run(ServiceControllerImpl.java:1556)
at
org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1985)
at
org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1487)
at
org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1364)
at java.lang.Thread.run(Thread.java:748)
{noformat}
Another server keeps blocked during joining:
{noformat}
"MSC service thread 1-2" #15 prio=5 os_prio=0 tid=0x00007fec98a02000 nid=0x2f9f
waiting on condition [0x00007fec8046c000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000000f6182fa8> (a
java.util.concurrent.CompletableFuture$Signaller)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1695)
at java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323)
at java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1775)
at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1915)
at org.infinispan.util.concurrent.CompletableFutures.await(CompletableFutures.java:93)
at org.infinispan.remoting.transport.Transport.invokeRemotely(Transport.java:72)
at
org.infinispan.topology.LocalTopologyManagerImpl.executeOnCoordinator(LocalTopologyManagerImpl.java:688)
at
org.infinispan.topology.LocalTopologyManagerImpl.join(LocalTopologyManagerImpl.java:148)
at
org.infinispan.statetransfer.StateTransferManagerImpl.start(StateTransferManagerImpl.java:109)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:79)
at org.infinispan.commons.util.SecurityActions$$Lambda$467/848998352.run(Unknown
Source)
at org.infinispan.commons.util.SecurityActions.doPrivileged(SecurityActions.java:71)
at
org.infinispan.commons.util.SecurityActions.invokeAccessibly(SecurityActions.java:76)
at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:185)
at
org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:968)
at
org.infinispan.factories.AbstractComponentRegistry.lambda$invokePrioritizedMethods$6(AbstractComponentRegistry.java:703)
at org.infinispan.factories.AbstractComponentRegistry$$Lambda$481/252402945.run(Unknown
Source)
at org.infinispan.factories.SecurityActions.lambda$run$1(SecurityActions.java:72)
at org.infinispan.factories.SecurityActions$$Lambda$482/1109883309.run(Unknown Source)
at org.infinispan.security.Security.doPrivileged(Security.java:49)
at org.infinispan.factories.SecurityActions.run(SecurityActions.java:71)
at
org.infinispan.factories.AbstractComponentRegistry.invokePrioritizedMethods(AbstractComponentRegistry.java:696)
at
org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:689)
at
org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:607)
at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:244)
at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1051)
at
org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:421)
at
org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:646)
at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:591)
at
org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:477)
at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:463)
at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:496)
at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:487)
at
org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:94)
at
org.jboss.as.clustering.infinispan.DefaultCacheContainer.getCache(DefaultCacheContainer.java:85)
at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26)
at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14)
at org.infinispan.security.Security.doPrivileged(Security.java:49)
at org.infinispan.server.hotrod.SecurityActions.doPrivileged(SecurityActions.java:36)
at org.infinispan.server.hotrod.SecurityActions.getCache(SecurityActions.java:53)
at org.infinispan.server.hotrod.HotRodServer.getCacheInstance(HotRodServer.java:450)
at org.infinispan.server.hotrod.HotRodServer.preStartCaches(HotRodServer.java:330)
at org.infinispan.server.hotrod.HotRodServer.startTransport(HotRodServer.java:314)
at
org.infinispan.server.core.AbstractProtocolServer.startInternal(AbstractProtocolServer.java:60)
at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:250)
at org.infinispan.server.hotrod.HotRodServer.startInternal(HotRodServer.java:106)
at
org.infinispan.server.core.AbstractProtocolServer.start(AbstractProtocolServer.java:78)
at
org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:136)
at
org.infinispan.server.endpoint.subsystem.SecurityActions$6.run(SecurityActions.java:133)
at org.infinispan.security.Security.doPrivileged(Security.java:44)
at
org.infinispan.server.endpoint.subsystem.SecurityActions.doPrivileged(SecurityActions.java:42)
at
org.infinispan.server.endpoint.subsystem.SecurityActions.startProtocolServer(SecurityActions.java:140)
at
org.infinispan.server.endpoint.subsystem.ProtocolServerService.startProtocolServer(ProtocolServerService.java:194)
at
org.infinispan.server.endpoint.subsystem.ProtocolServerService.start(ProtocolServerService.java:152)
- locked <0x00000000e0f59e90> (a
org.infinispan.server.endpoint.subsystem.ProtocolServerService)
at
org.jboss.msc.service.ServiceControllerImpl$StartTask.startService(ServiceControllerImpl.java:1736)
at
org.jboss.msc.service.ServiceControllerImpl$StartTask.execute(ServiceControllerImpl.java:1698)
at
org.jboss.msc.service.ServiceControllerImpl$ControllerTask.run(ServiceControllerImpl.java:1556)
at
org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1985)
at
org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1487)
at
org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1378)
at java.lang.Thread.run(Thread.java:748)
{noformat}
--
This message was sent by Atlassian JIRA
(v7.5.0#75005)
2791
days inactive
2791
days old
infinispan-issues@lists.jboss.org
0 comments
1 participants
participants (1)
-
Gustavo Fernandes (JIRA)