ISPN000136: Execution error: org.infinispan.CacheException: Member node-0/ejb no longer in cluster. Forcing tx rollback for GlobalTransaction:<node-0/ejb>:80:remote -------------------------------------------------------------------------------------------------------------------------------------------------------------------- Key: WFLY-961 URL: https://issues.jboss.org/browse/WFLY-961 Project: WildFly Issue Type: Bug Security Level: Public(Everyone can see) Components: Clustering Reporter: Radoslav Husar Assignee: Radoslav Husar We are sometimes seeing this in the TS runs, i.e. in this run: http://lightning.mw.lab.eng.bos.redhat.com/jenkins/job/as7-master-testsui... {noformat} 21:14:15,628 INFO [org.infinispan.config.ConfigurationValidatingVisitor] (MSC service thread 1-4) ISPN000152: Passivation configured without an eviction policy being selected. Only manually evicted entities will be pasivated. 21:14:15,650 INFO [org.infinispan.jmx.CacheJmxRegistration] (MSC service thread 1-4) ISPN000031: MBeans were successfully registered to the platform mbean server. 21:14:15,681 INFO [org.jboss.as.clustering.infinispan] (MSC service thread 1-4) JBAS010281: Started //default-host//stateful cache from web container 21:14:15,684 INFO [org.infinispan.configuration.cache.EvictionConfigurationBuilder] (MSC service thread 1-4) ISPN000152: Passivation configured without an eviction policy being selected. Only manually evicted entities will be pasivated. 21:14:15,687 INFO [org.infinispan.configuration.cache.EvictionConfigurationBuilder] (MSC service thread 1-4) ISPN000152: Passivation configured without an eviction policy being selected. Only manually evicted entities will be pasivated. 21:14:15,723 INFO [org.jboss.web] (MSC service thread 1-4) JBAS018210: Registering web context: /stateful 21:14:15,749 INFO [org.jboss.as.server] (management-handler-thread - 4) JBAS018559: Deployed "stateful.war" 21:14:15,886 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (Incoming-6,null) ISPN000136: Execution error: org.infinispan.CacheException: Member node-0/ejb no longer in cluster. Forcing tx rollback for GlobalTransaction:<node-0/ejb>:80:remote at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.abortIfRemoteTransactionInvalid(AbstractTxLockingInterceptor.java:115) at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:115) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:106) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.StateTransferLockInterceptor.handleWithRetries(StateTransferLockInterceptor.java:208) at org.infinispan.interceptors.StateTransferLockInterceptor.visitPrepareCommand(StateTransferLockInterceptor.java:85) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:130) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:89) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:70) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.commands.tx.PrepareCommand.perform(PrepareCommand.java:127) at org.infinispan.remoting.InboundInvocationHandlerImpl.handleInternal(InboundInvocationHandlerImpl.java:127) at org.infinispan.remoting.InboundInvocationHandlerImpl.handleWithWaitForBlocks(InboundInvocationHandlerImpl.java:136) at org.infinispan.remoting.InboundInvocationHandlerImpl.handleWithRetry(InboundInvocationHandlerImpl.java:162) at org.infinispan.remoting.InboundInvocationHandlerImpl.handle(InboundInvocationHandlerImpl.java:114) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommand(CommandAwareRpcDispatcher.java:221) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:201) at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:456) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:363) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:238) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:543) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.blocks.mux.MuxUpHandler.up(MuxUpHandler.java:130) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.JChannel.up(JChannel.java:716) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1026) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.RSVP.up(RSVP.java:192) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.FRAG2.up(FRAG2.java:181) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:418) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:400) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:889) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:244) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.UNICAST2.handleDataReceived(UNICAST2.java:793) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.UNICAST2.up(UNICAST2.java:365) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:602) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.BARRIER.up(BARRIER.java:102) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:143) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.FD.up(FD.java:273) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:288) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.MERGE2.up(MERGE2.java:205) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.Discovery.up(Discovery.java:359) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.stack.Protocol.up(Protocol.java:363) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.TP.passMessageUp(TP.java:1180) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1728) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1710) [jgroups-3.0.9.Final.jar:3.0.9.Final] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [rt.jar:1.6.0_29] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [rt.jar:1.6.0_29] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_29] 21:14:15,921 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (Incoming-6,null) ISPN000136: Execution error: org.infinispan.CacheException: Member node-0/ejb no longer in cluster. Forcing tx rollback for GlobalTransaction:<node-0/ejb>:83:remote at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.abortIfRemoteTransactionInvalid(AbstractTxLockingInterceptor.java:115) at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:115) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:58) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:106) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.StateTransferLockInterceptor.handleWithRetries(StateTransferLockInterceptor.java:208) at org.infinispan.interceptors.StateTransferLockInterceptor.visitPrepareCommand(StateTransferLockInterceptor.java:85) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:130) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:89) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:70) at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:113) at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.commands.tx.PrepareCommand.perform(PrepareCommand.java:127) at org.infinispan.remoting.InboundInvocationHandlerImpl.handleInternal(InboundInvocationHandlerImpl.java:127) at org.infinispan.remoting.InboundInvocationHandlerImpl.handleWithWaitForBlocks(InboundInvocationHandlerImpl.java:136) at org.infinispan.remoting.InboundInvocationHandlerImpl.handleWithRetry(InboundInvocationHandlerImpl.java:162) at org.infinispan.remoting.InboundInvocationHandlerImpl.handle(InboundInvocationHandlerImpl.java:114) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommand(CommandAwareRpcDispatcher.java:221) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:201) at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:456) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:363) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:238) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:543) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.blocks.mux.MuxUpHandler.up(MuxUpHandler.java:130) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.JChannel.up(JChannel.java:716) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1026) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.RSVP.up(RSVP.java:192) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.FRAG2.up(FRAG2.java:181) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:418) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:400) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:889) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:244) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.UNICAST2.handleDataReceived(UNICAST2.java:793) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.UNICAST2.up(UNICAST2.java:365) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:602) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.BARRIER.up(BARRIER.java:102) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:143) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.FD.up(FD.java:273) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:288) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.MERGE2.up(MERGE2.java:205) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.Discovery.up(Discovery.java:359) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.stack.Protocol.up(Protocol.java:363) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.TP.passMessageUp(TP.java:1180) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1728) [jgroups-3.0.9.Final.jar:3.0.9.Final] at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1710) [jgroups-3.0.9.Final.jar:3.0.9.Final] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [rt.jar:1.6.0_29] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [rt.jar:1.6.0_29] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_29] {noformat}