[jboss-jira] [JBoss JIRA] (WFLY-6426) ISE: Lock already released! after cluster-wide rebalance

Michal Vinkler created WFLY-6426: ------------------------------------ Summary: ISE: Lock already released! after cluster-wide rebalance Key: WFLY-6426 URL: https://issues.jboss.org/browse/WFLY-6426 Project: WildFly Issue Type: Bug Components: Clustering Reporter: Michal Vinkler Assignee: Paul Ferraro Seen in this scenario: failover-ejb-ejbremote-jvmkill-dist-sync After perf18 was killed, perf19 received new cluster view, started cluster-wide rebalance and then logged {{IllegalStateException: Lock already released!}}: {code:title=perf19} [JBossINF] [0m[31m08:33:22,291 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (remote-thread--p5-t31) ISPN000136: Error executing command LockControlCommand, writing keys []: java.lang.IllegalStateException: Lock already released! [JBossINF] at org.infinispan.util.concurrent.locks.impl.InfinispanLock$LockPlaceHolder.lock(InfinispanLock.java:324) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.lock(DefaultLockManager.java:234) [JBossINF] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockAllAndRecord(AbstractLockingInterceptor.java:199) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.checkPendingAndLockAllKeys(AbstractTxLockingInterceptor.java:199) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.lockAllOrRegisterBackupLock(AbstractTxLockingInterceptor.java:165) [JBossINF] at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitLockControlCommand(PessimisticLockingInterceptor.java:189) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:157) [JBossINF] at org.infinispan.interceptors.TxInterceptor.visitLockControlCommand(TxInterceptor.java:215) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitLockControlCommand(AbstractVisitor.java:163) [JBossINF] at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitLockControlCommand(TransactionSynchronizerInterceptor.java:75) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:228) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.visitLockControlCommand(StateTransferInterceptor.java:101) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:107) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.visitLockControlCommand(InvocationContextInterceptor.java:81) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitLockControlCommand(AbstractVisitor.java:163) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) [JBossINF] at org.infinispan.commands.control.LockControlCommand.perform(LockControlCommand.java:122) [JBossINF] at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokePerform(BasePerCacheInboundInvocationHandler.java:92) [JBossINF] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:34) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) {code} Causing perf20 to log {{EJB Invocation failed}}: {code:title=perf20} [JBossINF] [0m[31m08:33:22,660 ERROR [org.jboss.as.ejb3.invocation] (default task-61) WFLYEJB0034: EJB Invocation failed on component RemoteStatefulSBImpl for method public abstract int org.jboss.test.clusterbench.common.ejb.CommonStatefulSB.getSerialAndIncrement(): javax.ejb.EJBException: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from perf19, see cause for remote stack trace [JBossINF] at org.jboss.as.ejb3.tx.CMTTxInterceptor.handleExceptionInOurTx(CMTTxInterceptor.java:187) [JBossINF] at org.jboss.as.ejb3.tx.CMTTxInterceptor.invokeInOurTx(CMTTxInterceptor.java:277) [JBossINF] at org.jboss.as.ejb3.tx.CMTTxInterceptor.required(CMTTxInterceptor.java:327) [JBossINF] at org.jboss.as.ejb3.tx.CMTTxInterceptor.processInvocation(CMTTxInterceptor.java:239) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.as.ejb3.remote.EJBRemoteTransactionPropagatingInterceptor.processInvocation(EJBRemoteTransactionPropagatingInterceptor.java:79) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.as.ejb3.component.interceptors.CurrentInvocationContextInterceptor.processInvocation(CurrentInvocationContextInterceptor.java:41) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.as.ejb3.component.invocationmetrics.WaitTimeInterceptor.processInvocation(WaitTimeInterceptor.java:43) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.as.ejb3.security.SecurityContextInterceptor.processInvocation(SecurityContextInterceptor.java:100) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.as.ejb3.component.interceptors.ShutDownInterceptorFactory$1.processInvocation(ShutDownInterceptorFactory.java:64) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.as.ejb3.deployment.processors.EjbSuspendInterceptor.processInvocation(EjbSuspendInterceptor.java:53) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.as.ejb3.component.interceptors.LoggingInterceptor.processInvocation(LoggingInterceptor.java:66) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.as.ee.component.NamespaceContextInterceptor.processInvocation(NamespaceContextInterceptor.java:50) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.as.ejb3.component.interceptors.AdditionalSetupInterceptor.processInvocation(AdditionalSetupInterceptor.java:54) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.invocation.ContextClassLoaderInterceptor.processInvocation(ContextClassLoaderInterceptor.java:64) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:356) [JBossINF] at org.wildfly.security.manager.WildFlySecurityManager.doChecked(WildFlySecurityManager.java:636) [JBossINF] at org.jboss.invocation.AccessCheckingInterceptor.processInvocation(AccessCheckingInterceptor.java:61) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:356) [JBossINF] at org.jboss.invocation.PrivilegedWithCombinerInterceptor.processInvocation(PrivilegedWithCombinerInterceptor.java:80) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:340) [JBossINF] at org.jboss.invocation.ChainedInterceptor.processInvocation(ChainedInterceptor.java:61) [JBossINF] at org.jboss.as.ee.component.ViewService$View.invoke(ViewService.java:195) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.invokeMethod(MethodInvocationMessageHandler.java:327) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.access$100(MethodInvocationMessageHandler.java:67) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler$1.run(MethodInvocationMessageHandler.java:200) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.processMessage(MethodInvocationMessageHandler.java:262) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver.processMessage(VersionOneProtocolChannelReceiver.java:213) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versiontwo.VersionTwoProtocolChannelReceiver.processMessage(VersionTwoProtocolChannelReceiver.java:76) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver.handleMessage(VersionOneProtocolChannelReceiver.java:159) [JBossINF] at org.jboss.remoting3.remote.RemoteConnectionChannel$5.run(RemoteConnectionChannel.java:456) [JBossINF] at org.jboss.remoting3.EndpointImpl$TrackingExecutor$1.run(EndpointImpl.java:717) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) [JBossINF] Caused by: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from perf19, see cause for remote stack trace [JBossINF] at org.infinispan.remoting.transport.AbstractTransport.checkResponse(AbstractTransport.java:44) [JBossINF] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:760) [JBossINF] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$175(JGroupsTransport.java:612) [JBossINF] at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602) [JBossINF] at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) [JBossINF] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) [JBossINF] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) [JBossINF] at org.infinispan.remoting.transport.jgroups.RspListFuture.futureDone(RspListFuture.java:31) [JBossINF] at org.jgroups.blocks.Request.checkCompletion(Request.java:151) [JBossINF] at org.jgroups.blocks.GroupRequest.receiveResponse(GroupRequest.java:119) [JBossINF] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:369) [JBossINF] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:233) [JBossINF] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:695) [JBossINF] at org.jgroups.JChannel.up(JChannel.java:738) [JBossINF] at org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:120) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:379) [JBossINF] at org.jgroups.protocols.FORK.up(FORK.java:114) [JBossINF] at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) [JBossINF] at org.jgroups.protocols.FlowControl.up(FlowControl.java:392) [JBossINF] at org.jgroups.protocols.FlowControl.up(FlowControl.java:381) [JBossINF] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1043) [JBossINF] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) [JBossINF] at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1064) [JBossINF] at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:779) [JBossINF] at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:645) [JBossINF] at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:155) [JBossINF] at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:200) [JBossINF] at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:310) [JBossINF] at org.jgroups.protocols.MERGE3.up(MERGE3.java:285) [JBossINF] at org.jgroups.protocols.Discovery.up(Discovery.java:296) [JBossINF] at org.jgroups.protocols.TP.passMessageUp(TP.java:1567) [JBossINF] at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1783) [JBossINF] ... 3 more [JBossINF] Caused by: java.lang.IllegalStateException: Lock already released! [JBossINF] at org.infinispan.util.concurrent.locks.impl.InfinispanLock$LockPlaceHolder.lock(InfinispanLock.java:324) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.lock(DefaultLockManager.java:234) [JBossINF] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockAllAndRecord(AbstractLockingInterceptor.java:199) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.checkPendingAndLockAllKeys(AbstractTxLockingInterceptor.java:199) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.lockAllOrRegisterBackupLock(AbstractTxLockingInterceptor.java:165) [JBossINF] at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitLockControlCommand(PessimisticLockingInterceptor.java:189) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.TxInterceptor.invokeNextInterceptorAndVerifyTransaction(TxInterceptor.java:157) [JBossINF] at org.infinispan.interceptors.TxInterceptor.visitLockControlCommand(TxInterceptor.java:215) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitLockControlCommand(AbstractVisitor.java:163) [JBossINF] at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitLockControlCommand(TransactionSynchronizerInterceptor.java:75) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleTxCommand(StateTransferInterceptor.java:228) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.visitLockControlCommand(StateTransferInterceptor.java:101) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:107) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.visitLockControlCommand(InvocationContextInterceptor.java:81) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitLockControlCommand(AbstractVisitor.java:163) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) [JBossINF] at org.infinispan.commands.control.LockControlCommand.perform(LockControlCommand.java:122) [JBossINF] at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokePerform(BasePerCacheInboundInvocationHandler.java:92) [JBossINF] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:34) [JBossINF] ... 3 more {code} Causing client to get: {code} 2016/03/22 08:33:22:679 EDT [ERROR][Runner - 1091] HOST perf17.mw.lab.eng.bos.redhat.com:rootProcess:c - Error getting response. <javax.ejb.EJBException: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from perf19, see cause for remote stack trace> javax.ejb.EJBException: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from perf19, see cause for remote stack trace at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler$1.run(MethodInvocationMessageHandler.java:216) at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.processMessage(MethodInvocationMessageHandler.java:262) at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver.processMessage(VersionOneProtocolChannelReceiver.java:213) at org.jboss.as.ejb3.remote.protocol.versiontwo.VersionTwoProtocolChannelReceiver.processMessage(VersionTwoProtocolChannelReceiver.java:76) at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver.handleMessage(VersionOneProtocolChannelReceiver.java:159) at org.jboss.remoting3.remote.RemoteConnectionChannel$5.run(RemoteConnectionChannel.java:456) at org.jboss.remoting3.EndpointImpl$TrackingExecutor$1.run(EndpointImpl.java:717) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) at ...asynchronous invocation...(Unknown Source) at org.jboss.ejb.client.remoting.InvocationExceptionResponseHandler$MethodInvocationExceptionResultProducer.getResult(InvocationExceptionResponseHandler.java:96) at org.jboss.ejb.client.EJBClientInvocationContext$1.run(EJBClientInvocationContext.java:282) at java.security.AccessController.doPrivileged(Native Method) at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:279) at org.jboss.ejb.client.EJBObjectInterceptor.handleInvocationResult(EJBObjectInterceptor.java:64) at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:290) at org.jboss.ejb.client.EJBHomeInterceptor.handleInvocationResult(EJBHomeInterceptor.java:88) at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:290) at org.jboss.ejb.client.TransactionInterceptor.handleInvocationResult(TransactionInterceptor.java:46) at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:290) at org.jboss.ejb.client.ReceiverInterceptor.handleInvocationResult(ReceiverInterceptor.java:142) at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:265) at org.jboss.ejb.client.EJBClientInvocationContext.awaitResponse(EJBClientInvocationContext.java:453) at org.jboss.ejb.client.EJBInvocationHandler.doInvoke(EJBInvocationHandler.java:204) at org.jboss.ejb.client.EJBInvocationHandler.doInvoke(EJBInvocationHandler.java:183) at org.jboss.ejb.client.EJBInvocationHandler.invoke(EJBInvocationHandler.java:146) at com.sun.proxy.$Proxy2.getSerialAndIncrement(Unknown Source) at org.jboss.smartfrog.clustering.ejb3.StatefulSBProcessorFactoryImpl$EJB3RequestProcessor.processRequest(StatefulSBProcessorFactoryImpl.java:84) at org.jboss.smartfrog.loaddriver.CompoundRequestProcessorFactoryImpl$CompoundRequestProcessor.processRequest(CompoundRequestProcessorFactoryImpl.java:52) at org.jboss.smartfrog.loaddriver.Runner.run(Runner.java:103) at java.lang.Thread.run(Thread.java:745) {code} Server links: http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-7x-failover-ejb-e... http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-7x-failover-ejb-e... Client link: http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-7x-failover-ejb-e... -- This message was sent by Atlassian JIRA (v6.4.11#64026)