[jboss-jira] [JBoss JIRA] (WFLY-6176) ISPN000136 - This remote transaction ... is already rolled back

ISPN000136 - This remote transaction ... is already rolled back --------------------------------------------------------------- Key: WFLY-6176 URL: https://issues.jboss.org/browse/WFLY-6176 Project: WildFly Issue Type: Bug Components: Clustering Affects Versions: 10.0.0.Final Reporter: Michal Vinkler Assignee: Paul Ferraro Priority: Minor Fix For: 10.1.0.Final Seen in this scenario in ER5: * http-session-jvmkill-dist-async-3owners With no obvious connection to a failover of any node, perf21 logged this error and the following warning once: {code} [JBossINF] [0m[31m18:18:06,117 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (Incoming-4,ee,perf21) ISPN000136: Error executing command PrepareCommand, writing keys []: org.infinispan.transaction.xa.InvalidTransactionException: This remote transaction GlobalTransaction:<perf20>:38042:remote is already rolled back [JBossINF] at org.infinispan.transaction.impl.RemoteTransaction.checkIfRolledBack(RemoteTransaction.java:135) [JBossINF] at org.infinispan.transaction.impl.RemoteTransaction.putLookedUpEntry(RemoteTransaction.java:69) [JBossINF] at org.infinispan.context.impl.AbstractTxInvocationContext.putLookedUpEntry(AbstractTxInvocationContext.java:101) [JBossINF] at org.infinispan.container.EntryFactoryImpl.wrapEntryForWriting(EntryFactoryImpl.java:141) [JBossINF] at org.infinispan.interceptors.EntryWrappingInterceptor$EntryWrappingVisitor.visitPutKeyValueCommand(EntryWrappingInterceptor.java:617) [JBossINF] at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:74) [JBossINF] at org.infinispan.interceptors.EntryWrappingInterceptor.wrapEntriesForPrepare(EntryWrappingInterceptor.java:677) [JBossINF] at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:104) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:36) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:90) [JBossINF] at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitPrepareCommand(PessimisticLockingInterceptor.java:99) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:157) [JBossINF] at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:144) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112) [JBossINF] at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:39) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:228) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.visitPrepareCommand(StateTransferInterceptor.java:86) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:107) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:76) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.perform(PrepareCommand.java:97) [JBossINF] at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokePerform(BasePerCacheInboundInvocationHandler.java:92) [JBossINF] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:34) [JBossINF] at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.handleRunnable(BasePerCacheInboundInvocationHandler.java:132) [JBossINF] at org.infinispan.remoting.inboundhandler.NonTotalOrderTxPerCacheInboundInvocationHandler.handle(NonTotalOrderTxPerCacheInboundInvocationHandler.java:99) [JBossINF] at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleCacheRpcCommand(GlobalInboundInvocationHandler.java:125) [JBossINF] at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:74) [JBossINF] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommandFromLocalCluster(CommandAwareRpcDispatcher.java:179) [JBossINF] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:149) [JBossINF] at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:460) [JBossINF] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:377) [JBossINF] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:250) [JBossINF] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:684) [JBossINF] at org.jgroups.JChannel.up(JChannel.java:760) [JBossINF] at org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:156) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.protocols.FORK.up(FORK.java:161) [JBossINF] at org.jgroups.protocols.FRAG2.up(FRAG2.java:182) [JBossINF] at org.jgroups.protocols.FlowControl.up(FlowControl.java:447) [JBossINF] at org.jgroups.protocols.FlowControl.up(FlowControl.java:447) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:294) [JBossINF] at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:474) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK2.deliverBatch(NAKACK2.java:982) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK2.removeAndPassUp(NAKACK2.java:912) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK2.handleMessages(NAKACK2.java:884) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:703) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:213) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.protocols.TP.passBatchUp(TP.java:1610) [JBossINF] at org.jgroups.protocols.TP$BatchHandler.run(TP.java:1860) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) [JBossINF] [JBossINF] [0m[33m18:18:06,121 WARN [org.infinispan.remoting.inboundhandler.NonTotalOrderTxPerCacheInboundInvocationHandler] (Incoming-4,ee,perf21) ISPN000071: Caught exception when handling command PrepareCommand {modifications=[PutKeyValueCommand{key=eIJgDI6uD1pzq9CxTLHP1Ba6gJ6PKXm_Ef_4L5AI, value=[B@617cf71b, flags=[IGNORE_RETURN_VALUES], putIfAbsent=false, valueMatcher=MATCH_ALWAYS, metadata=EmbeddedExpirableMetadata{lifespan=-1, maxIdle=-1, version=null}, successful=true}], onePhaseCommit=true, retried=false, gtx=GlobalTransaction:<perf20>:38042:remote, cacheName='clusterbench-ee7.ear.clusterbench-ee7-web-default.war', topologyId=10}: org.infinispan.transaction.xa.InvalidTransactionException: This remote transaction GlobalTransaction:<perf20>:38042:remote is already rolled back [JBossINF] at org.infinispan.transaction.impl.RemoteTransaction.checkIfRolledBack(RemoteTransaction.java:135) [JBossINF] at org.infinispan.transaction.impl.RemoteTransaction.putLookedUpEntry(RemoteTransaction.java:69) [JBossINF] at org.infinispan.context.impl.AbstractTxInvocationContext.putLookedUpEntry(AbstractTxInvocationContext.java:101) [JBossINF] at org.infinispan.container.EntryFactoryImpl.wrapEntryForWriting(EntryFactoryImpl.java:141) [JBossINF] at org.infinispan.interceptors.EntryWrappingInterceptor$EntryWrappingVisitor.visitPutKeyValueCommand(EntryWrappingInterceptor.java:617) [JBossINF] at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:74) [JBossINF] at org.infinispan.interceptors.EntryWrappingInterceptor.wrapEntriesForPrepare(EntryWrappingInterceptor.java:677) [JBossINF] at org.infinispan.interceptors.EntryWrappingInterceptor.visitPrepareCommand(EntryWrappingInterceptor.java:104) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.NotificationInterceptor.visitPrepareCommand(NotificationInterceptor.java:36) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.invokeNextAndCommitIf1Pc(AbstractTxLockingInterceptor.java:90) [JBossINF] at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitPrepareCommand(PessimisticLockingInterceptor.java:99) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:157) [JBossINF] at org.infinispan.interceptors.TxInterceptor.visitPrepareCommand(TxInterceptor.java:144) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112) [JBossINF] at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitPrepareCommand(TransactionSynchronizerInterceptor.java:39) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:228) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.visitPrepareCommand(StateTransferInterceptor.java:86) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:107) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:76) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitPrepareCommand(AbstractVisitor.java:112) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:173) [JBossINF] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) [JBossINF] at org.infinispan.commands.tx.PrepareCommand.perform(PrepareCommand.java:97) [JBossINF] at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokePerform(BasePerCacheInboundInvocationHandler.java:92) [JBossINF] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:34) [JBossINF] at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.handleRunnable(BasePerCacheInboundInvocationHandler.java:132) [JBossINF] at org.infinispan.remoting.inboundhandler.NonTotalOrderTxPerCacheInboundInvocationHandler.handle(NonTotalOrderTxPerCacheInboundInvocationHandler.java:99) [JBossINF] at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleCacheRpcCommand(GlobalInboundInvocationHandler.java:125) [JBossINF] at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:74) [JBossINF] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommandFromLocalCluster(CommandAwareRpcDispatcher.java:179) [JBossINF] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:149) [JBossINF] at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:460) [JBossINF] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:377) [JBossINF] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:250) [JBossINF] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:684) [JBossINF] at org.jgroups.JChannel.up(JChannel.java:760) [JBossINF] at org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:156) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.protocols.FORK.up(FORK.java:161) [JBossINF] at org.jgroups.protocols.FRAG2.up(FRAG2.java:182) [JBossINF] at org.jgroups.protocols.FlowControl.up(FlowControl.java:447) [JBossINF] at org.jgroups.protocols.FlowControl.up(FlowControl.java:447) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:294) [JBossINF] at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:474) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK2.deliverBatch(NAKACK2.java:982) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK2.removeAndPassUp(NAKACK2.java:912) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK2.handleMessages(NAKACK2.java:884) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:703) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:213) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:420) [JBossINF] at org.jgroups.protocols.TP.passBatchUp(TP.java:1610) [JBossINF] at org.jgroups.protocols.TP$BatchHandler.run(TP.java:1860) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) {code} Link: http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-7x-failover-http-... ISPN-4124 seems to be related but not assigned to anyone.