Radoslav Husar created WFLY-11241: ------------------------------------- Summary: CoarseWebFailoverTestCase.test fails intermittently (after Infinispan 9.4.0 upgrade?) Key: WFLY-11241 URL: https://issues.jboss.org/browse/WFLY-11241 Project: WildFly Issue Type: Bug Components: Clustering Reporter: Radoslav Husar Assignee: Paul Ferraro Started to fail intermittently again after 19 Oct, which seems to correlate with Infinispan 9.4.0.Final upgrade on 17 Oct. {code} 07:17:11,909 INFO [io.undertow.servlet] (default task-1) /CoarseWebFailoverTestCase/simple, value = 3 07:17:12,491 INFO [org.infinispan.CLUSTER] (remote-thread--p6-t2) [Context=CoarseWebFailoverTestCase.war] ISPN100009: Advancing to rebalance phase READ_NEW_WRITE_ALL, topology id 33 07:17:12,503 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (remote-thread--p6-t3) ISPN000136: Error executing command LockControlCommand, writing keys []: org.infinispan.util.concurrent.TimeoutException: Could not acquire lock on SessionCreationMetaDataKey(nUIcjLVQdeTRWxsBIwHRSRarTu75v5FgXeZQ4Lnt) in behalf of transaction GlobalTx:node-3:24. Current owner GlobalTx:node-2:30. at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.timeout(DefaultPendingLockManager.java:252) at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.awaitOn(DefaultPendingLockManager.java:337) at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.awaitPendingTransactionsForAllKeys(DefaultPendingLockManager.java:137) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.checkPendingAndLockAllKeys(AbstractTxLockingInterceptor.java:153) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.lockAllOrRegisterBackupLock(AbstractTxLockingInterceptor.java:122) at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.localLockCommandWork(PessimisticLockingInterceptor.java:208) at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.lambda$new$0(PessimisticLockingInterceptor.java:46) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenApply(BaseAsyncInterceptor.java:81) at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitLockControlCommand(PessimisticLockingInterceptor.java:191) at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:117) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:183) at org.infinispan.interceptors.impl.TxInterceptor.visitLockControlCommand(TxInterceptor.java:223) at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:117) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54) at org.infinispan.interceptors.BaseAsyncInterceptor.lambda$new$0(BaseAsyncInterceptor.java:22) at org.infinispan.interceptors.InvocationSuccessFunction.apply(InvocationSuccessFunction.java:25) at org.infinispan.interceptors.impl.SimpleAsyncInvocationStage.addCallback(SimpleAsyncInvocationStage.java:70) at org.infinispan.interceptors.InvocationStage.thenApply(InvocationStage.java:45) at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:224) at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitCommand(TransactionSynchronizerInterceptor.java:46) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:185) at org.infinispan.statetransfer.StateTransferInterceptor.visitLockControlCommand(StateTransferInterceptor.java:90) at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:117) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:123) at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:56) at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54) at org.infinispan.interceptors.DDAsyncInterceptor.visitLockControlCommand(DDAsyncInterceptor.java:160) at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:117) at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:50) at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:234) at org.infinispan.commands.control.LockControlCommand.invokeAsync(LockControlCommand.java:126) at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokeCommand(BasePerCacheInboundInvocationHandler.java:95) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.invoke(BaseBlockingRunnable.java:99) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.runAsync(BaseBlockingRunnable.java:71) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:40) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at org.wildfly.clustering.service.concurrent.ClassLoaderThreadFactory.lambda$newThread$0(ClassLoaderThreadFactory.java:47) at java.lang.Thread.run(Thread.java:748) 07:17:12,511 WARN [org.infinispan.remoting.inboundhandler.NonTotalOrderTxPerCacheInboundInvocationHandler] (remote-thread--p6-t3) ISPN000071: Caught exception when handling command LockControlCommand{cache=CoarseWebFailoverTestCase.war, keys=[SessionCreationMetaDataKey(nUIcjLVQdeTRWxsBIwHRSRarTu75v5FgXeZQ4Lnt)], flags=[ZERO_LOCK_ACQUISITION_TIMEOUT, FORCE_WRITE_LOCK], unlock=false, gtx=GlobalTx:node-3:24}: org.infinispan.util.concurrent.TimeoutException: Could not acquire lock on SessionCreationMetaDataKey(nUIcjLVQdeTRWxsBIwHRSRarTu75v5FgXeZQ4Lnt) in behalf of transaction GlobalTx:node-3:24. Current owner GlobalTx:node-2:30. at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.timeout(DefaultPendingLockManager.java:252) at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.awaitOn(DefaultPendingLockManager.java:337) at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.awaitPendingTransactionsForAllKeys(DefaultPendingLockManager.java:137) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.checkPendingAndLockAllKeys(AbstractTxLockingInterceptor.java:153) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.lockAllOrRegisterBackupLock(AbstractTxLockingInterceptor.java:122) at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.localLockCommandWork(PessimisticLockingInterceptor.java:208) at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.lambda$new$0(PessimisticLockingInterceptor.java:46) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenApply(BaseAsyncInterceptor.java:81) at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitLockControlCommand(PessimisticLockingInterceptor.java:191) at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:117) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:183) at org.infinispan.interceptors.impl.TxInterceptor.visitLockControlCommand(TxInterceptor.java:223) at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:117) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54) at org.infinispan.interceptors.BaseAsyncInterceptor.lambda$new$0(BaseAsyncInterceptor.java:22) at org.infinispan.interceptors.InvocationSuccessFunction.apply(InvocationSuccessFunction.java:25) at org.infinispan.interceptors.impl.SimpleAsyncInvocationStage.addCallback(SimpleAsyncInvocationStage.java:70) at org.infinispan.interceptors.InvocationStage.thenApply(InvocationStage.java:45) at org.infinispan.interceptors.BaseAsyncInterceptor.asyncInvokeNext(BaseAsyncInterceptor.java:224) at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitCommand(TransactionSynchronizerInterceptor.java:46) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:185) at org.infinispan.statetransfer.StateTransferInterceptor.visitLockControlCommand(StateTransferInterceptor.java:90) at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:117) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:123) at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:56) at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54) at org.infinispan.interceptors.DDAsyncInterceptor.visitLockControlCommand(DDAsyncInterceptor.java:160) at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:117) at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:50) at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:234) at org.infinispan.commands.control.LockControlCommand.invokeAsync(LockControlCommand.java:126) at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokeCommand(BasePerCacheInboundInvocationHandler.java:95) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.invoke(BaseBlockingRunnable.java:99) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.runAsync(BaseBlockingRunnable.java:71) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:40) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at org.wildfly.clustering.service.concurrent.ClassLoaderThreadFactory.lambda$newThread$0(ClassLoaderThreadFactory.java:47) at java.lang.Thread.run(Thread.java:748) 07:17:13,090 INFO [org.infinispan.CLUSTER] (remote-thread--p6-t3) [Context=CoarseWebFailoverTestCase.war] ISPN100010: Finished rebalance with members [node-3, node-2], topology id 34 07:17:14,298 INFO [io.undertow.servlet] (default task-2) /CoarseWebFailoverTestCase/simple, value = 4 {code} https://gist.github.com/rhusar/496f1c7c5a308db1901a81a4437b570e https://ci.wildfly.org/viewLog.html?buildId=125879&tab=buildResultsDi... -- This message was sent by Atlassian Jira (v7.12.1#712002)