Persistence (Blocking) thread pool should have a queue ------------------------------------------------------ Key: ISPN-10364 URL: https://issues.jboss.org/browse/ISPN-10364 Project: Infinispan Issue Type: Bug Components: Core Reporter: Will Burns Assignee: Will Burns Priority: Major Fix For: 10.0.0.Beta4 When Persistence executor was added it didn't have a queue. Now that more things are being run on this executor we can end up running things on the wrong thread. This is apparent with this error. {code} 16:36:47,118 ERROR (testng-Test:[]) [TestSuiteProgress] Test failed: org.infinispan.distribution.groups.TransactionalGetGroupKeysTest.testRemoveGroupKeysWithPersistenceAndPassivation[PRIMARY_OWNER, DIST_SYNC, TO=true, isolation=READ_COMMITTED] org.infinispan.commons.CacheException: Could not commit implicit transaction at org.infinispan.cache.impl.CacheImpl.tryCommit(CacheImpl.java:1991) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.transactionalRemoveGroup(CacheImpl.java:651) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.removeGroup(CacheImpl.java:634) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.removeGroup(CacheImpl.java:627) ~[classes/:?] at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.removeGroup(AbstractDelegatingAdvancedCache.java:342) ~[classes/:?] at org.infinispan.distribution.groups.GetGroupKeysTest.testRemoveGroupKeysWithPersistenceAndPassivation(GetGroupKeysTest.java:165) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?] at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:124) ~[testng-6.14.3.jar:?] at org.testng.internal.MethodInvocationHelper$1.runTestMethod(MethodInvocationHelper.java:230) ~[testng-6.14.3.jar:?] at org.infinispan.commons.test.TestNGLongTestsHook.run(TestNGLongTestsHook.java:24) ~[infinispan-commons-test-10.0.0-SNAPSHOT.jar:10.0.0-SNAPSHOT] at org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:242) ~[testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeMethod(Invoker.java:579) [testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:719) [testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:989) [testng-6.14.3.jar:?] at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:125) [testng-6.14.3.jar:?] at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:109) [testng-6.14.3.jar:?] at org.testng.TestRunner.privateRun(TestRunner.java:648) [testng-6.14.3.jar:?] at org.testng.TestRunner.run(TestRunner.java:505) [testng-6.14.3.jar:?] at org.testng.SuiteRunner.runTest(SuiteRunner.java:455) [testng-6.14.3.jar:?] at org.testng.SuiteRunner.access$000(SuiteRunner.java:40) [testng-6.14.3.jar:?] at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:489) [testng-6.14.3.jar:?] at org.testng.internal.thread.ThreadUtil$1.call(ThreadUtil.java:52) [testng-6.14.3.jar:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:834) [?:?] Caused by: javax.transaction.RollbackException: ARJUNA016053: Could not commit transaction. at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1300) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at org.infinispan.cache.impl.CacheImpl.tryCommit(CacheImpl.java:1988) ~[classes/:?] ... 28 more Suppressed: javax.transaction.xa.XAException at org.infinispan.transaction.impl.TransactionCoordinator.handleCommitFailure(TransactionCoordinator.java:215) ~[classes/:?] at org.infinispan.transaction.impl.TransactionCoordinator.commit(TransactionCoordinator.java:161) ~[classes/:?] at org.infinispan.transaction.xa.XaTransactionTable.commit(XaTransactionTable.java:119) ~[classes/:?] at org.infinispan.transaction.xa.TransactionXaAdapter.commit(TransactionXaAdapter.java:68) ~[classes/:?] at com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord.topLevelOnePhaseCommit(XAResourceRecord.java:702) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.coordinator.BasicAction.onePhaseCommit(BasicAction.java:2386) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.coordinator.BasicAction.End(BasicAction.java:1497) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:96) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1288) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at org.infinispan.cache.impl.CacheImpl.tryCommit(CacheImpl.java:1988) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.transactionalRemoveGroup(CacheImpl.java:651) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.removeGroup(CacheImpl.java:634) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.removeGroup(CacheImpl.java:627) ~[classes/:?] at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.removeGroup(AbstractDelegatingAdvancedCache.java:342) ~[classes/:?] at org.infinispan.distribution.groups.GetGroupKeysTest.testRemoveGroupKeysWithPersistenceAndPassivation(GetGroupKeysTest.java:165) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?] at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:124) ~[testng-6.14.3.jar:?] at org.testng.internal.MethodInvocationHelper$1.runTestMethod(MethodInvocationHelper.java:230) ~[testng-6.14.3.jar:?] at org.infinispan.commons.test.TestNGLongTestsHook.run(TestNGLongTestsHook.java:24) ~[infinispan-commons-test-10.0.0-SNAPSHOT.jar:10.0.0-SNAPSHOT] at org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:242) ~[testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeMethod(Invoker.java:579) [testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:719) [testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:989) [testng-6.14.3.jar:?] at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:125) [testng-6.14.3.jar:?] at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:109) [testng-6.14.3.jar:?] at org.testng.TestRunner.privateRun(TestRunner.java:648) [testng-6.14.3.jar:?] at org.testng.TestRunner.run(TestRunner.java:505) [testng-6.14.3.jar:?] at org.testng.SuiteRunner.runTest(SuiteRunner.java:455) [testng-6.14.3.jar:?] at org.testng.SuiteRunner.access$000(SuiteRunner.java:40) [testng-6.14.3.jar:?] at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:489) [testng-6.14.3.jar:?] at org.testng.internal.thread.ThreadUtil$1.call(ThreadUtil.java:52) [testng-6.14.3.jar:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:834) [?:?] Caused by: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from Test-NodeB-15612, see cause for remote stack trace at org.infinispan.remoting.transport.ResponseCollectors.wrapRemoteException(ResponseCollectors.java:28) ~[classes/:?] at org.infinispan.remoting.transport.impl.MapResponseCollector.addException(MapResponseCollector.java:64) ~[classes/:?] at org.infinispan.remoting.transport.impl.MapResponseCollector$IgnoreLeavers.addException(MapResponseCollector.java:102) ~[classes/:?] at org.infinispan.remoting.transport.ValidResponseCollector.addResponse(ValidResponseCollector.java:29) ~[classes/:?] at org.infinispan.remoting.transport.impl.MultiTargetRequest.onResponse(MultiTargetRequest.java:91) ~[classes/:?] at org.infinispan.remoting.transport.impl.RequestRepository.addResponse(RequestRepository.java:52) ~[classes/:?] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processResponse(JGroupsTransport.java:1406) ~[classes/:?] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1309) ~[classes/:?] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$300(JGroupsTransport.java:130) ~[classes/:?] at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1454) ~[classes/:?] at org.jgroups.JChannel.up(JChannel.java:775) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:916) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.RSVP.up(RSVP.java:164) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.FRAG3.up(FRAG3.java:166) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:338) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:338) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.tom.TOA.up(TOA.java:119) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:848) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:241) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1019) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:745) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:400) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:590) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.Discovery.up(Discovery.java:296) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.protocols.TP.passMessageUp(TP.java:1265) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] at org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:87) ~[jgroups-4.1.1.Final.jar:4.1.1.Final] ... 3 more Suppressed: org.infinispan.util.logging.TraceException at org.infinispan.interceptors.impl.SimpleAsyncInvocationStage.get(SimpleAsyncInvocationStage.java:41) ~[classes/:?] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:120) ~[classes/:?] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:137) ~[classes/:?] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:126) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.base.CommandInterceptor.visitCommand(CommandInterceptor.java:163) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:61) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.prepareHandler(EntryWrappingInterceptor.java:193) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.wrapEntriesForPrepareAndApply(EntryWrappingInterceptor.java:882) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:186) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.visitPrepareCommand(DDAsyncInterceptor.java:131) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) ~[classes/:?] at org.infinispan.interceptors.impl.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:41) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) ~[classes/:?] at org.infinispan.interceptors.impl.TxInterceptor.handlePrepareCommand(TxInterceptor.java:158) ~[classes/:?] at org.infinispan.interceptors.impl.TxInterceptor.visitPrepareCommand(TxInterceptor.java:127) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:155) ~[classes/:?] at org.infinispan.interceptors.totalorder.TotalOrderInterceptor.visitPrepareCommand(TotalOrderInterceptor.java:69) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) ~[classes/:?] at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitCommand(TransactionSynchronizerInterceptor.java:41) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:130) ~[classes/:?] at org.infinispan.interceptors.totalorder.TotalOrderStateTransferInterceptor.localPrepare(TotalOrderStateTransferInterceptor.java:66) ~[classes/:?] at org.infinispan.interceptors.totalorder.TotalOrderStateTransferInterceptor.visitPrepareCommand(TotalOrderStateTransferInterceptor.java:31) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.visitPrepareCommand(DDAsyncInterceptor.java:131) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:128) ~[classes/:?] at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:89) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:61) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.visitPrepareCommand(DDAsyncInterceptor.java:131) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:49) ~[classes/:?] at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:244) ~[classes/:?] at org.infinispan.transaction.impl.TransactionCoordinator.commit(TransactionCoordinator.java:159) ~[classes/:?] at org.infinispan.transaction.xa.XaTransactionTable.commit(XaTransactionTable.java:119) ~[classes/:?] at org.infinispan.transaction.xa.TransactionXaAdapter.commit(TransactionXaAdapter.java:68) ~[classes/:?] at com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord.topLevelOnePhaseCommit(XAResourceRecord.java:702) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.coordinator.BasicAction.onePhaseCommit(BasicAction.java:2386) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.coordinator.BasicAction.End(BasicAction.java:1497) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:96) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1288) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at org.infinispan.cache.impl.CacheImpl.tryCommit(CacheImpl.java:1988) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.transactionalRemoveGroup(CacheImpl.java:651) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.removeGroup(CacheImpl.java:634) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.removeGroup(CacheImpl.java:627) ~[classes/:?] at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.removeGroup(AbstractDelegatingAdvancedCache.java:342) ~[classes/:?] at org.infinispan.distribution.groups.GetGroupKeysTest.testRemoveGroupKeysWithPersistenceAndPassivation(GetGroupKeysTest.java:165) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?] at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:124) ~[testng-6.14.3.jar:?] at org.testng.internal.MethodInvocationHelper$1.runTestMethod(MethodInvocationHelper.java:230) ~[testng-6.14.3.jar:?] at org.infinispan.commons.test.TestNGLongTestsHook.run(TestNGLongTestsHook.java:24) ~[infinispan-commons-test-10.0.0-SNAPSHOT.jar:10.0.0-SNAPSHOT] at org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:242) ~[testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeMethod(Invoker.java:579) [testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:719) [testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:989) [testng-6.14.3.jar:?] at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:125) [testng-6.14.3.jar:?] at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:109) [testng-6.14.3.jar:?] at org.testng.TestRunner.privateRun(TestRunner.java:648) [testng-6.14.3.jar:?] at org.testng.TestRunner.run(TestRunner.java:505) [testng-6.14.3.jar:?] at org.testng.SuiteRunner.runTest(SuiteRunner.java:455) [testng-6.14.3.jar:?] at org.testng.SuiteRunner.access$000(SuiteRunner.java:40) [testng-6.14.3.jar:?] at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:489) [testng-6.14.3.jar:?] at org.testng.internal.thread.ThreadUtil$1.call(ThreadUtil.java:52) [testng-6.14.3.jar:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:834) [?:?] Suppressed: org.infinispan.util.logging.TraceException at org.infinispan.interceptors.impl.SimpleAsyncInvocationStage.get(SimpleAsyncInvocationStage.java:41) ~[classes/:?] at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:246) ~[classes/:?] at org.infinispan.transaction.impl.TransactionCoordinator.commit(TransactionCoordinator.java:159) ~[classes/:?] at org.infinispan.transaction.xa.XaTransactionTable.commit(XaTransactionTable.java:119) ~[classes/:?] at org.infinispan.transaction.xa.TransactionXaAdapter.commit(TransactionXaAdapter.java:68) ~[classes/:?] at com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord.topLevelOnePhaseCommit(XAResourceRecord.java:702) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.coordinator.BasicAction.onePhaseCommit(BasicAction.java:2386) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.coordinator.BasicAction.End(BasicAction.java:1497) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:96) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1288) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126) ~[narayana-jta-5.9.0.Final.jar:5.9.0.Final (revision: f0935)] at org.infinispan.cache.impl.CacheImpl.tryCommit(CacheImpl.java:1988) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.transactionalRemoveGroup(CacheImpl.java:651) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.removeGroup(CacheImpl.java:634) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.removeGroup(CacheImpl.java:627) ~[classes/:?] at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.removeGroup(AbstractDelegatingAdvancedCache.java:342) ~[classes/:?] at org.infinispan.distribution.groups.GetGroupKeysTest.testRemoveGroupKeysWithPersistenceAndPassivation(GetGroupKeysTest.java:165) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?] at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:124) ~[testng-6.14.3.jar:?] at org.testng.internal.MethodInvocationHelper$1.runTestMethod(MethodInvocationHelper.java:230) ~[testng-6.14.3.jar:?] at org.infinispan.commons.test.TestNGLongTestsHook.run(TestNGLongTestsHook.java:24) ~[infinispan-commons-test-10.0.0-SNAPSHOT.jar:10.0.0-SNAPSHOT] at org.testng.internal.MethodInvocationHelper.invokeHookable(MethodInvocationHelper.java:242) ~[testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeMethod(Invoker.java:579) [testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:719) [testng-6.14.3.jar:?] at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:989) [testng-6.14.3.jar:?] at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:125) [testng-6.14.3.jar:?] at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:109) [testng-6.14.3.jar:?] at org.testng.TestRunner.privateRun(TestRunner.java:648) [testng-6.14.3.jar:?] at org.testng.TestRunner.run(TestRunner.java:505) [testng-6.14.3.jar:?] at org.testng.SuiteRunner.runTest(SuiteRunner.java:455) [testng-6.14.3.jar:?] at org.testng.SuiteRunner.access$000(SuiteRunner.java:40) [testng-6.14.3.jar:?] at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:489) [testng-6.14.3.jar:?] at org.testng.internal.thread.ThreadUtil$1.call(ThreadUtil.java:52) [testng-6.14.3.jar:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:834) [?:?] Caused by: org.infinispan.commons.CacheException: Problems invoking command. at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.exceptionHandlingCommand(BasePerCacheInboundInvocationHandler.java:155) ~[classes/:?] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.afterCommandException(BaseBlockingRunnable.java:150) ~[classes/:?] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.runSync(BaseBlockingRunnable.java:59) ~[classes/:?] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:39) ~[classes/:?] ... 3 more Caused by: java.lang.AssertionError: Thread name is: remote-thread-Test-NodeB-p3529-t2 at org.infinispan.persistence.manager.PersistenceManagerImpl.deleteFromAllStoresSync(PersistenceManagerImpl.java:606) ~[classes/:?] at org.infinispan.eviction.impl.ActivationManagerImpl.onRemove(ActivationManagerImpl.java:103) ~[classes/:?] at org.infinispan.container.impl.AbstractInternalDataContainer.lambda$remove$1(AbstractInternalDataContainer.java:205) ~[classes/:?] at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$remap$16(BoundedLocalCache.java:2199) ~[caffeine-2.6.2.jar:?] at java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1908) ~[?:?] at com.github.benmanes.caffeine.cache.BoundedLocalCache.remap(BoundedLocalCache.java:2194) ~[caffeine-2.6.2.jar:?] at com.github.benmanes.caffeine.cache.BoundedLocalCache.compute(BoundedLocalCache.java:2146) ~[caffeine-2.6.2.jar:?] at com.github.benmanes.caffeine.cache.LocalCache.compute(LocalCache.java:100) ~[caffeine-2.6.2.jar:?] at org.infinispan.container.impl.AbstractInternalDataContainer.remove(AbstractInternalDataContainer.java:204) ~[classes/:?] at org.infinispan.container.entries.ReadCommittedEntry.commit(ReadCommittedEntry.java:141) ~[classes/:?] at org.infinispan.statetransfer.CommitManager.commitEntry(CommitManager.java:130) ~[classes/:?] at org.infinispan.statetransfer.CommitManager.commit(CommitManager.java:98) ~[classes/:?] at org.infinispan.interceptors.locking.ClusteringDependentLogic$DistributionLogic.commitSingleEntry(ClusteringDependentLogic.java:627) ~[classes/:?] at org.infinispan.interceptors.locking.ClusteringDependentLogic$AbstractClusteringDependentLogic.lambda$commitEntry$2(ClusteringDependentLogic.java:238) ~[classes/:?] at org.infinispan.commons.util.concurrent.CallerRunsRejectOnShutdownPolicy.rejectedExecution(CallerRunsRejectOnShutdownPolicy.java:19) ~[infinispan-commons-10.0.0-SNAPSHOT.jar:10.0.0-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:825) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1355) ~[?:?] at org.infinispan.executors.LazyInitializingExecutorService.execute(LazyInitializingExecutorService.java:138) ~[classes/:?] at org.infinispan.interceptors.locking.ClusteringDependentLogic$AbstractClusteringDependentLogic.commitEntry(ClusteringDependentLogic.java:238) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitContextEntry(EntryWrappingInterceptor.java:602) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitEntryIfNeeded(EntryWrappingInterceptor.java:847) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitContextEntries(EntryWrappingInterceptor.java:588) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.lambda$new$2(EntryWrappingInterceptor.java:153) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenApply(BaseAsyncInterceptor.java:86) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.prepareHandler(EntryWrappingInterceptor.java:191) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.applyModificationsAndThen(EntryWrappingInterceptor.java:905) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.wrapEntriesForPrepareAndApply(EntryWrappingInterceptor.java:869) ~[classes/:?] at org.infinispan.interceptors.impl.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:186) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.visitPrepareCommand(DDAsyncInterceptor.java:131) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) ~[classes/:?] at org.infinispan.interceptors.impl.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:41) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:188) ~[classes/:?] at org.infinispan.interceptors.impl.TxInterceptor.handlePrepareCommand(TxInterceptor.java:138) ~[classes/:?] at org.infinispan.interceptors.impl.TxInterceptor.visitPrepareCommand(TxInterceptor.java:127) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:155) ~[classes/:?] at org.infinispan.interceptors.totalorder.TotalOrderInterceptor.visitPrepareCommand(TotalOrderInterceptor.java:95) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:232) ~[classes/:?] at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitCommand(TransactionSynchronizerInterceptor.java:46) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:61) ~[classes/:?] at org.infinispan.interceptors.totalorder.TotalOrderStateTransferInterceptor.remotePrepare(TotalOrderStateTransferInterceptor.java:55) ~[classes/:?] at org.infinispan.interceptors.totalorder.TotalOrderStateTransferInterceptor.visitPrepareCommand(TotalOrderStateTransferInterceptor.java:33) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.visitPrepareCommand(DDAsyncInterceptor.java:131) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:128) ~[classes/:?] at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:89) ~[classes/:?] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:61) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.visitPrepareCommand(DDAsyncInterceptor.java:131) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:202) ~[classes/:?] at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:49) ~[classes/:?] at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:226) ~[classes/:?] at org.infinispan.commands.tx.PrepareCommand.invokeAsync(PrepareCommand.java:122) ~[classes/:?] at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokeCommand(BasePerCacheInboundInvocationHandler.java:118) ~[classes/:?] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.runSync(BaseBlockingRunnable.java:55) ~[classes/:?] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:39) ~[classes/:?] {code}