[infinispan-issues] [JBoss JIRA] (ISPN-4614) DistFineGrainedAtomicMapAPITest.testConcurrentWritesOnExistingMap fails randomly

DistFineGrainedAtomicMapAPITest.testConcurrentWritesOnExistingMap fails randomly -------------------------------------------------------------------------------- Key: ISPN-4614 URL: https://issues.jboss.org/browse/ISPN-4614 Project: Infinispan Issue Type: Bug Components: Test Suite - Core Reporter: William Burns Assignee: Radim Vansa Priority: Blocker Labels: testsuite_stability Test failure: http://ci.infinispan.org/viewLog.html?buildId=10595&tab=buildResultsD... {code} java.util.concurrent.TimeoutException at java.util.concurrent.FutureTask.get(FutureTask.java:201) at org.infinispan.atomic.BaseAtomicHashMapAPITest.testConcurrentWritesOnExistingMap(BaseAtomicHashMapAPITest.java:196) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84) at org.testng.internal.Invoker.invokeMethod(Invoker.java:714) at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901) at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231) at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127) at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111) at org.testng.TestRunner.privateRun(TestRunner.java:767) at org.testng.TestRunner.run(TestRunner.java:617) at org.testng.SuiteRunner.runTest(SuiteRunner.java:348) at org.testng.SuiteRunner.access$000(SuiteRunner.java:38) at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:382) at org.testng.internal.thread.ThreadUtil$2.call(ThreadUtil.java:64) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:724) ------- Stdout: ------- 03:09:01,622 WARN (testng-DistWriteSkewAtomicMapAPITest:) [arjuna] ARJUNA012125: TwoPhaseCoordinator.beforeCompletion - failed for SynchronizationImple< 0:ffff0ab30531:d19c:53e2e839:48d78, SynchronizationAdapter{localTransaction=LocalTransaction{remoteLockedNodes=null, isMarkedForRollback=false, lockedKeys=null, backupKeyLocks=[MagicKey#null{63b22462@DistWriteSkewAtomicMapAPITest-NodeA-14450/7}], topologyId=4, stateTransferFlag=null} org.infinispan.transaction.synchronization.SyncLocalTransaction@16dd9} org.infinispan.transaction.synchronization.SynchronizationAdapter@16df8 > org.infinispan.commons.CacheException: Could not prepare. at org.infinispan.transaction.synchronization.SynchronizationAdapter.beforeCompletion(SynchronizationAdapter.java:46) at com.arjuna.ats.internal.jta.resources.arjunacore.SynchronizationImple.beforeCompletion(SynchronizationImple.java:76) at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.beforeCompletion(TwoPhaseCoordinator.java:273) at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:93) at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162) at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1165) at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126) at org.infinispan.atomic.DistWriteSkewAtomicMapAPITest.doWriteSkewTest(DistWriteSkewAtomicMapAPITest.java:87) at org.infinispan.atomic.DistWriteSkewAtomicMapAPITest.testWriteSkewOnBackupOwner(DistWriteSkewAtomicMapAPITest.java:37) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84) at org.testng.internal.Invoker.invokeMethod(Invoker.java:714) at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901) at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231) at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127) at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111) at org.testng.TestRunner.privateRun(TestRunner.java:767) at org.testng.TestRunner.run(TestRunner.java:617) at org.testng.SuiteRunner.runTest(SuiteRunner.java:348) at org.testng.SuiteRunner.access$000(SuiteRunner.java:38) at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:382) at org.testng.internal.thread.ThreadUtil$2.call(ThreadUtil.java:64) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:724) Caused by: javax.transaction.xa.XAException at org.infinispan.transaction.impl.TransactionCoordinator.prepare(TransactionCoordinator.java:141) at org.infinispan.transaction.impl.TransactionCoordinator.prepare(TransactionCoordinator.java:104) at org.infinispan.transaction.synchronization.SynchronizationAdapter.beforeCompletion(SynchronizationAdapter.java:44) ... 28 more Caused by: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from DistWriteSkewAtomicMapAPITest-NodeA-14450, see cause for remote stack trace at org.infinispan.remoting.transport.AbstractTransport.checkResponse(AbstractTransport.java:41) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:362) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:165) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:535) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:281) at org.infinispan.interceptors.distribution.VersionedDistributionInterceptor.prepareOnAffectedNodes(VersionedDistributionInterceptor.java:35) at org.infinispan.interceptors.distribution.TxDistributionInterceptor.visitPrepareCommand(TxDistributionInterceptor.java:197) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.VersionedEntryWrappingInterceptor.visitPrepareCommand(VersionedEntryWrappingInterceptor.java:51) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:78) at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:87) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:36) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:124) at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:111) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:97) at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:42) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:217) at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:147) at org.infinispan.statetransfer.StateTransferInterceptor.visitPrepareCommand(StateTransferInterceptor.java:75) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:97) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:111) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:74) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:97) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:333) at org.infinispan.transaction.impl.TransactionCoordinator.prepare(TransactionCoordinator.java:121) ... 30 more Caused by: org.infinispan.transaction.WriteSkewException: Write skew detected on key MagicKey#null{63b22462@DistWriteSkewAtomicMapAPITest-NodeA-14450/7} for transaction null at org.infinispan[testng-DistFineGrainedAtomicMapAPITest] Test testConcurrentWritesOnExistingMap(org.infinispan.atomic.DistFineGrainedAtomicMapAPITest) failed. .transaction.impl.WriteSkewHelper.performWriteSkewCheckAndReturnNewVersions(WriteSkewHelper.java:54) at org.infinispan.interceptors.locking.ClusteringDependentLogic$AbstractClusteringDependentLogic.clusteredCreateNewVersionsAndCheckForWriteSkews(ClusteringDependentLogic.java:155) at org.infinispan.interceptors.locking.ClusteringDependentLogic$AbstractClusteringDependentLogic.createNewVersionsAndCheckForWriteSkews(ClusteringDependentLogic.java:93) at org.infinispan.interceptors.VersionedEntryWrappingInterceptor.visitPrepareCommand(VersionedEntryWrappingInterceptor.java:53) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:78) at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:87) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:36) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:124) at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:111) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:97) at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:39) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:217) at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:147) at org.infinispan.statetransfer.StateTransferInterceptor.visitPrepareCommand(StateTransferInterceptor.java:75) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:97) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:111) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:74) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:97) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:124) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:333) at org.infinispan.commands.tx.PrepareCommand.perform(PrepareCommand.java:114) at org.infinispan.remoting.InboundInvocationHandlerImpl.handleInternal(InboundInvocationHandlerImpl.java:95) at org.infinispan.remoting.InboundInvocationHandlerImpl.access$000(InboundInvocationHandlerImpl.java:50) at org.infinispan.remoting.InboundInvocationHandlerImpl$2.run(InboundInvocationHandlerImpl.java:181) ... 3 more 03:09:07,924 ERROR (testng-DistFineGrainedAtomicMapAPITest:) [UnitTestTestNGListener] Test testConcurrentWritesOnExistingMap(org.infinispan.atomic.DistFineGrainedAtomicMapAPITest) failed. java.util.concurrent.TimeoutException at java.util.concurrent.FutureTask.get(FutureTask.java:201) at org.infinispan.atomic.BaseAtomicHashMapAPITest.testConcurrentWritesOnExistingMap(BaseAtomicHashMapAPITest.java:196) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84) at org.testng.internal.Invoker.invokeMethod(Invoker.java:714) at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901) at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231) at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127) at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111) at org.testng.TestRunner.privateRun(TestRunner.java:767) at org.testng.TestRunner.run(TestRunner.java:617) at org.testng.SuiteRunner.runTest(SuiteRunner.java:348) at org.testng.SuiteRunner.access$000(SuiteRunner.java:38) at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:382) at org.testng.internal.thread.ThreadUtil$2.call(ThreadUtil.java:64) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:724) Test suite progress: tests succeeded: 3746, failed: 2, skipped: 0. 03:09:01,621 ERROR (ForkThread-2,DistFineGrainedAtomicMapAPITest:) [InvocationContextInterceptor] ISPN000136: Execution error org.infinispan.util.concurrent.TimeoutException: ISPN000299: Unable to acquire lock after 2 seconds for key DeltaCompositeKey[deltaAwareValueKey=testConcurrentReadsOnExistingMap, entryKey=[the-2]] and requestor GlobalTransaction:<DistFineGrainedAtomicMapAPITest-NodeA-15358>:93611:local. Lock is held by GlobalTransaction:<DistFineGrainedAtomicMapAPITest-NodeA-15358>:93607:local, while request came from null at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:198) at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLockNoCheck(LockManagerImpl.java:181) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockKey(AbstractLockingInterceptor.java:127) at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitApplyDeltaCommand(PessimisticLockingInterceptor.java:159) at org.infinispan.commands.write.ApplyDeltaCommand.acceptVisitor(ApplyDeltaCommand.java:86) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitApplyDeltaCommand(AbstractVisitor.java:63) at org.infinispan.commands.write.ApplyDeltaCommand.acceptVisitor(ApplyDeltaCommand.java:86) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.TxInterceptor.enlistWriteAndInvokeNext(TxInterceptor.java:306) at org.infinispan.interceptors.TxInterceptor.visitApplyDeltaCommand(TxInterceptor.java:235) at org.infinispan.commands.write.ApplyDeltaCommand.acceptVisitor(ApplyDeltaCommand.java:86) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitApplyDeltaCommand(AbstractVisitor.java:63) at org.infinispan.commands.write.ApplyDeltaCommand.acceptVisitor(ApplyDeltaCommand.java:86) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:159) at org.infinispan.statetransfer.StateTransferInterceptor.visitApplyDeltaCommand(StateTransferInterceptor.java:105) at org.infinispan.commands.write.ApplyDeltaCommand.acceptVisitor(ApplyDeltaCommand.java:86) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitApplyDeltaCommand(AbstractVisitor.java:63) at org.infinispan.commands.write.ApplyDeltaCommand.acceptVisitor(ApplyDeltaCommand.java:86) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:111) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:74) at org.infinispan.commands.AbstractVisitor.visitApplyDeltaCommand(AbstractVisitor.java:63) at org.infinispan.commands.write.ApplyDeltaCommand.acceptVisitor(ApplyDeltaCommand.java:86) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:333) at org.infinispan.cache.impl.CacheImpl.applyDelta(CacheImpl.java:751) at org.infinispan.atomic.impl.FineGrainedAtomicHashMapProxy.invokeApplyDelta(FineGrainedAtomicHashMapProxy.java:275) at org.infinispan.atomic.impl.FineGrainedAtomicHashMapProxy.put(FineGrainedAtomicHashMapProxy.java:208) at org.infinispan.atomic.BaseAtomicHashMapAPITest$2.call(BaseAtomicHashMapAPITest.java:190) at org.infinispan.atomic.BaseAtomicHashMapAPITest$2.call(BaseAtomicHashMapAPITest.java:184) at org.infinispan.test.TestingUtil$1.call(TestingUtil.java:1277) at org.infinispan.test.AbstractInfinispanTest$LoggingCallable.call(AbstractInfinispanTest.java:424) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:724) 03:09:12,032 ERROR (ForkThread-2,DistRepeatableReadFineGrainedAtomicMapAPITest:) [InvocationContextInterceptor] ISPN000136: Execution error org.infinispan.util.concurrent.TimeoutException: Timed out waiting for 15 seconds for valid responses from any of [Sender{address=DistRepeatableReadFineGrainedAtomicMapAPITest-NodeB-5782, responded=false}]. at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher$FutureCollator.getResponseList(CommandAwareRpcDispatcher.java:531) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processCalls(CommandAwareRpcDispatcher.java:416) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommands(CommandAwareRpcDispatcher.java:130) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:538) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:281) at org.infinispan.interceptors.distribution.BaseDistributionInterceptor.invokeClusterGetCommandRemotely(BaseDistributionInterceptor.java:158) at org.infinispan.interceptors.distribution.BaseDistributionInterceptor.retrieveFromRemoteSource(BaseDistributionInterceptor.java:146) at org.infinispan.interceptors.distribution.TxDistributionInterceptor.remoteGet(TxDistributionInterceptor.java:328) at org.infinispan.interceptors.distribution.TxDistributionInterceptor.visitGetCommand(TxDistributionInterceptor.java:144) at org.infinispan.interceptors.distribution.TxDistributionInterceptor.visitGetKeyValueCommand(TxDistributionInterceptor.java:126) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:44) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.EntryWrappingInterceptor.visitGetKeyValueCommand(EntryWrappingInterceptor.java:129) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:44) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitGetKeyValueCommand(PessimisticLockingInterceptor.java:72) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:44) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:75) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:44) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.TxInterceptor.enlistReadAndInvokeNext(TxInterceptor.java:285) at org.infinispan.interceptors.TxInterceptor.visitGetKeyValueCommand(TxInterceptor.java:280) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:44) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:75) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:44) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:217) at org.infinispan.statetransfer.StateTransferInterceptor.handleDefault(StateTransferInterceptor.java:201) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:75) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:44) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.CacheMgmtInterceptor.visitGetKeyValueCommand(CacheMgmtInterceptor.java:92) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:44) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:111) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:74) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:75) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:44) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:333) at org.infinispan.cache.impl.CacheImpl.get(CacheImpl.java:392) at org.infinispan.cache.impl.CacheImpl.get(CacheImpl.java:384) at org.infinispan.atomic.impl.AtomicHashMapProxy.getDeltaMapForRead(AtomicHashMapProxy.java:73) at org.infinispan.atomic.impl.FineGrainedAtomicHashMapProxy.keySet(FineGrainedAtomicHashMapProxy.java:67) at org.infinispan.atomic.BaseAtomicHashMapAPITest$4.call(BaseAtomicHashMapAPITest.java:227) at org.infinispan.atomic.BaseAtomicHashMapAPITest$4.call(BaseAtomicHashMapAPITest.java:222) at org.infinispan.test.AbstractInfinispanTest$LoggingCallable.call(AbstractInfinispanTest.java:424) at org.infinispan.test.AbstractInfinispanTest$ConcurrentCallable.call(AbstractInfinispanTest.java:379) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:724) [testng-DistRepeatableReadFineGrainedAtomicMapAPITest] Test testConcurrentWritesAndIteration(org.infinispan.atomic.DistRepeatableReadFineGrainedAtomicMapAPITest) failed. {code}