Sessions sometimes fail to passivate with unable to acquire lock after with 0 milliseconds timeout -------------------------------------------------------------------------------------------------- Key: AS7-4728 URL: https://issues.jboss.org/browse/AS7-4728 Project: Application Server 7 Issue Type: Bug Components: Clustering Affects Versions: 7.1.1.Final Reporter: Radoslav Husar Assignee: Paul Ferraro Fix For: 7.1.3.Final (EAP) This is part of a long running test (soak test). There are no node failures so this is not anticipated. Repeating lock operation with 0 milliseconds 3x is suspicious to begin with. {noformat} 19:07:02,005 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (ContainerBackgroundProcessor[StandardEngine[jboss.web]]) ISPN000136: Execution error: org.infinispan.util.concurrent.TimeoutException: Unable to acquire lock after [0 milliseconds] on key [ULshl+wHh0iUAr33vnqM2H8E] for requestor [Thread[ContainerBackgroundProcessor[StandardEngine[jboss.web]],5,main]]! Lock held by [GlobalTransaction:<perf20/web>:612729:remote] at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:216) at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLockNoCheck(LockManagerImpl.java:199) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockKey(AbstractLockingInterceptor.java:114) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.visitEvictCommand(AbstractTxLockingInterceptor.java:82) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.MarshalledValueInterceptor.visitEvictCommand(MarshalledValueInterceptor.java:188) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.CacheMgmtInterceptor.visitEvictCommand(CacheMgmtInterceptor.java:86) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:131) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:90) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.CacheImpl.evict(CacheImpl.java:382) at org.infinispan.CacheImpl.evict(CacheImpl.java:375) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:303) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:300) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$ForceSynchronousCacheInvoker.invoke(DistributedCacheManager.java:526) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.invoke(DistributedCacheManager.java:494) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:307) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:320) at org.jboss.as.web.session.DistributableSessionManager.processUnloadedSessionPassivation(DistributableSessionManager.java:337) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.access$700(DistributableSessionManager.java:84) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.passivate(DistributableSessionManager.java:1556) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.access$300(DistributableSessionManager.java:1526) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.processExpirationPassivation(DistributableSessionManager.java:1290) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.AbstractSessionManager.processExpires(AbstractSessionManager.java:136) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.apache.catalina.session.ManagerBase.backgroundProcess(ManagerBase.java:375) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase.backgroundProcess(ContainerBase.java:1317) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1605) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1591) [jbossweb-7.0.15.Final.jar:] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_30] 19:07:02,021 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (ContainerBackgroundProcessor[StandardEngine[jboss.web]]) ISPN000136: Execution error: org.infinispan.util.concurrent.TimeoutException: Unable to acquire lock after [0 milliseconds] on key [ULshl+wHh0iUAr33vnqM2H8E] for requestor [Thread[ContainerBackgroundProcessor[StandardEngine[jboss.web]],5,main]]! Lock held by [GlobalTransaction:<perf20/web>:612729:remote] at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:216) at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLockNoCheck(LockManagerImpl.java:199) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockKey(AbstractLockingInterceptor.java:114) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.visitEvictCommand(AbstractTxLockingInterceptor.java:82) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.MarshalledValueInterceptor.visitEvictCommand(MarshalledValueInterceptor.java:188) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.CacheMgmtInterceptor.visitEvictCommand(CacheMgmtInterceptor.java:86) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:131) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:90) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.CacheImpl.evict(CacheImpl.java:382) at org.infinispan.CacheImpl.evict(CacheImpl.java:375) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:303) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:300) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$ForceSynchronousCacheInvoker.invoke(DistributedCacheManager.java:526) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.invoke(DistributedCacheManager.java:494) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:307) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:320) at org.jboss.as.web.session.DistributableSessionManager.processUnloadedSessionPassivation(DistributableSessionManager.java:337) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.access$700(DistributableSessionManager.java:84) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.passivate(DistributableSessionManager.java:1556) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.access$300(DistributableSessionManager.java:1526) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.processExpirationPassivation(DistributableSessionManager.java:1290) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.AbstractSessionManager.processExpires(AbstractSessionManager.java:136) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.apache.catalina.session.ManagerBase.backgroundProcess(ManagerBase.java:375) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase.backgroundProcess(ContainerBase.java:1317) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1605) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1591) [jbossweb-7.0.15.Final.jar:] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_30] 19:07:02,126 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (ContainerBackgroundProcessor[StandardEngine[jboss.web]]) ISPN000136: Execution error: org.infinispan.util.concurrent.TimeoutException: Unable to acquire lock after [0 milliseconds] on key [ULshl+wHh0iUAr33vnqM2H8E] for requestor [Thread[ContainerBackgroundProcessor[StandardEngine[jboss.web]],5,main]]! Lock held by [GlobalTransaction:<perf20/web>:612729:remote] at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:216) at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLockNoCheck(LockManagerImpl.java:199) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockKey(AbstractLockingInterceptor.java:114) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.visitEvictCommand(AbstractTxLockingInterceptor.java:82) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.MarshalledValueInterceptor.visitEvictCommand(MarshalledValueInterceptor.java:188) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.CacheMgmtInterceptor.visitEvictCommand(CacheMgmtInterceptor.java:86) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:131) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:90) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.CacheImpl.evict(CacheImpl.java:382) at org.infinispan.CacheImpl.evict(CacheImpl.java:375) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:303) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:300) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$ForceSynchronousCacheInvoker.invoke(DistributedCacheManager.java:526) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.invoke(DistributedCacheManager.java:494) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:307) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:320) at org.jboss.as.web.session.DistributableSessionManager.processUnloadedSessionPassivation(DistributableSessionManager.java:337) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.access$700(DistributableSessionManager.java:84) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.passivate(DistributableSessionManager.java:1556) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.access$300(DistributableSessionManager.java:1526) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.processExpirationPassivation(DistributableSessionManager.java:1290) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.AbstractSessionManager.processExpires(AbstractSessionManager.java:136) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.apache.catalina.session.ManagerBase.backgroundProcess(ManagerBase.java:375) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase.backgroundProcess(ContainerBase.java:1317) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1605) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1591) [jbossweb-7.0.15.Final.jar:] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_30] 19:07:02,138 ERROR [org.apache.catalina.session.ManagerBase] (ContainerBackgroundProcessor[StandardEngine[jboss.web]]) JBAS018079: Failed to passivate unloaded session ULshl+wHh0iUAr33vnqM2H8E: java.lang.RuntimeException: JBAS010296: Aborting cache operation after 3 retries. at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:82) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$ForceSynchronousCacheInvoker.invoke(DistributedCacheManager.java:526) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.invoke(DistributedCacheManager.java:494) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:307) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:320) at org.jboss.as.web.session.DistributableSessionManager.processUnloadedSessionPassivation(DistributableSessionManager.java:337) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.access$700(DistributableSessionManager.java:84) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.passivate(DistributableSessionManager.java:1556) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.access$300(DistributableSessionManager.java:1526) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.processExpirationPassivation(DistributableSessionManager.java:1290) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.AbstractSessionManager.processExpires(AbstractSessionManager.java:136) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.apache.catalina.session.ManagerBase.backgroundProcess(ManagerBase.java:375) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase.backgroundProcess(ContainerBase.java:1317) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1605) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1591) [jbossweb-7.0.15.Final.jar:] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_30] Caused by: org.infinispan.util.concurrent.TimeoutException: Unable to acquire lock after [0 milliseconds] on key [ULshl+wHh0iUAr33vnqM2H8E] for requestor [Thread[ContainerBackgroundProcessor[StandardEngine[jboss.web]],5,main]]! Lock held by [GlobalTransaction:<perf20/web>:612729:remote] at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:216) at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLockNoCheck(LockManagerImpl.java:199) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockKey(AbstractLockingInterceptor.java:114) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.visitEvictCommand(AbstractTxLockingInterceptor.java:82) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.MarshalledValueInterceptor.visitEvictCommand(MarshalledValueInterceptor.java:188) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.CacheMgmtInterceptor.visitEvictCommand(CacheMgmtInterceptor.java:86) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:131) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:90) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.CacheImpl.evict(CacheImpl.java:382) at org.infinispan.CacheImpl.evict(CacheImpl.java:375) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:303) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:300) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:60) ... 17 more {noformat}

Sessions sometimes fail to passivate with unable to acquire lock after with 0 milliseconds timeout -------------------------------------------------------------------------------------------------- Key: AS7-4728 URL: https://issues.jboss.org/browse/AS7-4728 Project: Application Server 7 Issue Type: Bug Components: Clustering Affects Versions: 7.1.1.Final Reporter: Radoslav Husar Assignee: Paul Ferraro Fix For: 7.2.0.Alpha1, 7.1.4.Final (EAP) This is part of a long running test (soak test). There are no node failures so this is not anticipated. Repeating lock operation with 0 milliseconds 3x is suspicious to begin with. {noformat} 19:07:02,005 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (ContainerBackgroundProcessor[StandardEngine[jboss.web]]) ISPN000136: Execution error: org.infinispan.util.concurrent.TimeoutException: Unable to acquire lock after [0 milliseconds] on key [ULshl+wHh0iUAr33vnqM2H8E] for requestor [Thread[ContainerBackgroundProcessor[StandardEngine[jboss.web]],5,main]]! Lock held by [GlobalTransaction:<perf20/web>:612729:remote] at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:216) at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLockNoCheck(LockManagerImpl.java:199) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockKey(AbstractLockingInterceptor.java:114) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.visitEvictCommand(AbstractTxLockingInterceptor.java:82) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.MarshalledValueInterceptor.visitEvictCommand(MarshalledValueInterceptor.java:188) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.CacheMgmtInterceptor.visitEvictCommand(CacheMgmtInterceptor.java:86) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:131) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:90) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.CacheImpl.evict(CacheImpl.java:382) at org.infinispan.CacheImpl.evict(CacheImpl.java:375) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:303) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:300) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$ForceSynchronousCacheInvoker.invoke(DistributedCacheManager.java:526) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.invoke(DistributedCacheManager.java:494) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:307) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:320) at org.jboss.as.web.session.DistributableSessionManager.processUnloadedSessionPassivation(DistributableSessionManager.java:337) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.access$700(DistributableSessionManager.java:84) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.passivate(DistributableSessionManager.java:1556) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.access$300(DistributableSessionManager.java:1526) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.processExpirationPassivation(DistributableSessionManager.java:1290) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.AbstractSessionManager.processExpires(AbstractSessionManager.java:136) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.apache.catalina.session.ManagerBase.backgroundProcess(ManagerBase.java:375) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase.backgroundProcess(ContainerBase.java:1317) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1605) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1591) [jbossweb-7.0.15.Final.jar:] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_30] 19:07:02,021 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (ContainerBackgroundProcessor[StandardEngine[jboss.web]]) ISPN000136: Execution error: org.infinispan.util.concurrent.TimeoutException: Unable to acquire lock after [0 milliseconds] on key [ULshl+wHh0iUAr33vnqM2H8E] for requestor [Thread[ContainerBackgroundProcessor[StandardEngine[jboss.web]],5,main]]! Lock held by [GlobalTransaction:<perf20/web>:612729:remote] at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:216) at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLockNoCheck(LockManagerImpl.java:199) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockKey(AbstractLockingInterceptor.java:114) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.visitEvictCommand(AbstractTxLockingInterceptor.java:82) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.MarshalledValueInterceptor.visitEvictCommand(MarshalledValueInterceptor.java:188) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.CacheMgmtInterceptor.visitEvictCommand(CacheMgmtInterceptor.java:86) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:131) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:90) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.CacheImpl.evict(CacheImpl.java:382) at org.infinispan.CacheImpl.evict(CacheImpl.java:375) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:303) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:300) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$ForceSynchronousCacheInvoker.invoke(DistributedCacheManager.java:526) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.invoke(DistributedCacheManager.java:494) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:307) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:320) at org.jboss.as.web.session.DistributableSessionManager.processUnloadedSessionPassivation(DistributableSessionManager.java:337) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.access$700(DistributableSessionManager.java:84) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.passivate(DistributableSessionManager.java:1556) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.access$300(DistributableSessionManager.java:1526) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.processExpirationPassivation(DistributableSessionManager.java:1290) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.AbstractSessionManager.processExpires(AbstractSessionManager.java:136) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.apache.catalina.session.ManagerBase.backgroundProcess(ManagerBase.java:375) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase.backgroundProcess(ContainerBase.java:1317) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1605) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1591) [jbossweb-7.0.15.Final.jar:] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_30] 19:07:02,126 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (ContainerBackgroundProcessor[StandardEngine[jboss.web]]) ISPN000136: Execution error: org.infinispan.util.concurrent.TimeoutException: Unable to acquire lock after [0 milliseconds] on key [ULshl+wHh0iUAr33vnqM2H8E] for requestor [Thread[ContainerBackgroundProcessor[StandardEngine[jboss.web]],5,main]]! Lock held by [GlobalTransaction:<perf20/web>:612729:remote] at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:216) at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLockNoCheck(LockManagerImpl.java:199) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockKey(AbstractLockingInterceptor.java:114) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.visitEvictCommand(AbstractTxLockingInterceptor.java:82) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.MarshalledValueInterceptor.visitEvictCommand(MarshalledValueInterceptor.java:188) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.CacheMgmtInterceptor.visitEvictCommand(CacheMgmtInterceptor.java:86) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:131) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:90) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.CacheImpl.evict(CacheImpl.java:382) at org.infinispan.CacheImpl.evict(CacheImpl.java:375) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:303) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:300) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$ForceSynchronousCacheInvoker.invoke(DistributedCacheManager.java:526) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.invoke(DistributedCacheManager.java:494) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:307) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:320) at org.jboss.as.web.session.DistributableSessionManager.processUnloadedSessionPassivation(DistributableSessionManager.java:337) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.access$700(DistributableSessionManager.java:84) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.passivate(DistributableSessionManager.java:1556) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.access$300(DistributableSessionManager.java:1526) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.processExpirationPassivation(DistributableSessionManager.java:1290) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.AbstractSessionManager.processExpires(AbstractSessionManager.java:136) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.apache.catalina.session.ManagerBase.backgroundProcess(ManagerBase.java:375) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase.backgroundProcess(ContainerBase.java:1317) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1605) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1591) [jbossweb-7.0.15.Final.jar:] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_30] 19:07:02,138 ERROR [org.apache.catalina.session.ManagerBase] (ContainerBackgroundProcessor[StandardEngine[jboss.web]]) JBAS018079: Failed to passivate unloaded session ULshl+wHh0iUAr33vnqM2H8E: java.lang.RuntimeException: JBAS010296: Aborting cache operation after 3 retries. at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:82) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$ForceSynchronousCacheInvoker.invoke(DistributedCacheManager.java:526) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.invoke(DistributedCacheManager.java:494) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:307) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager.evictSession(DistributedCacheManager.java:320) at org.jboss.as.web.session.DistributableSessionManager.processUnloadedSessionPassivation(DistributableSessionManager.java:337) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.access$700(DistributableSessionManager.java:84) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.passivate(DistributableSessionManager.java:1556) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager$PassivationCheck.access$300(DistributableSessionManager.java:1526) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.DistributableSessionManager.processExpirationPassivation(DistributableSessionManager.java:1290) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.jboss.as.web.session.AbstractSessionManager.processExpires(AbstractSessionManager.java:136) [jboss-as-web-7.1.2.Final-SNAPSHOT.jar:7.1.2.Final-SNAPSHOT] at org.apache.catalina.session.ManagerBase.backgroundProcess(ManagerBase.java:375) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase.backgroundProcess(ContainerBase.java:1317) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1605) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.processChildren(ContainerBase.java:1617) [jbossweb-7.0.15.Final.jar:] at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1591) [jbossweb-7.0.15.Final.jar:] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_30] Caused by: org.infinispan.util.concurrent.TimeoutException: Unable to acquire lock after [0 milliseconds] on key [ULshl+wHh0iUAr33vnqM2H8E] for requestor [Thread[ContainerBackgroundProcessor[StandardEngine[jboss.web]],5,main]]! Lock held by [GlobalTransaction:<perf20/web>:612729:remote] at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:216) at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLockNoCheck(LockManagerImpl.java:199) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockKey(AbstractLockingInterceptor.java:114) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.visitEvictCommand(AbstractTxLockingInterceptor.java:82) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.MarshalledValueInterceptor.visitEvictCommand(MarshalledValueInterceptor.java:188) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.CacheMgmtInterceptor.visitEvictCommand(CacheMgmtInterceptor.java:86) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:131) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:90) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) at org.infinispan.commands.AbstractVisitor.visitEvictCommand(AbstractVisitor.java:87) at org.infinispan.commands.write.EvictCommand.acceptVisitor(EvictCommand.java:47) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.CacheImpl.evict(CacheImpl.java:382) at org.infinispan.CacheImpl.evict(CacheImpl.java:375) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.infinispan.AbstractDelegatingCache.evict(AbstractDelegatingCache.java:60) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:303) at org.jboss.as.clustering.web.infinispan.DistributedCacheManager$4.invoke(DistributedCacheManager.java:300) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:60) ... 17 more {noformat}