Application hangs when the owners leave the cluster when clustered maxIdle expiration is enabled ------------------------------------------------------------------------------------------------ Key: ISPN-9446 URL: https://issues.redhat.com/browse/ISPN-9446 Project: Infinispan Issue Type: Bug Components: Expiration Reporter: Diego Lovison Assignee: Will Burns Priority: Critical Fix For: 9.4.0.Final 1) Execute the following application 3 times. 2) Wait until the application receives all topologies changes 3) See what application is returning null values. Let's say APP_3 4) Identify the other two application that are returning nonnull values. Let's say APP_1 and APP_2 5) The first line of APP_1 and APP_2 show the pid number 6) Execute the following command "kill -STOP $PID_APP_1 && kill -STOP $PID_APP_2" 7) Look in APP_3 until receiving a new topology and wait 20 seconds. 8) Execute the following command "kill -CONT $PID_APP_1 && kill -CONT $PID_APP_2" The APP_1 and APP_2 hang and don't continue anymore. Here is the stack trace from APP_1 {noformat} 8-08-22 11:45:06,537 INFO [org.infinispan.CLUSTER] (jgroups-3,Diegos-MacBook-Pro-29329:[]) ISPN000094: Received new cluster view for channel WeatherApp: [Diegos-MacBook-Pro-21527|3] (2) [Diegos-MacBook-Pro-21527, Diegos-MacBook-Pro-29329] 2018-08-22 11:45:06,542 INFO [org.infinispan.CLUSTER] (jgroups-3,Diegos-MacBook-Pro-29329:[]) ISPN100001: Node Diegos-MacBook-Pro-45456 left the cluster EventImpl{type=VIEW_CHANGED, newMembers=[Diegos-MacBook-Pro-21527, Diegos-MacBook-Pro-29329], oldMembers=[Diegos-MacBook-Pro-21527, Diegos-MacBook-Pro-45456, Diegos-MacBook-Pro-29329], localAddress=Diegos-MacBook-Pro-29329, viewId=3, subgroupsMerged=null, mergeView=false} 2018-08-22 11:45:16,548 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (remote-thread--p2-t1:[]) ISPN000136: Error executing command RemoveExpiredCommand, writing keys [Brazil] org.infinispan.util.concurrent.TimeoutException: ISPN000299: Unable to acquire lock after 10 seconds for key Brazil and requestor CommandInvocation:Diegos-MacBook-Pro-45456:1. Lock is held by CommandInvocation:Diegos-MacBook-Pro-29329:1 at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.get(DefaultLockManager.java:288) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.get(DefaultLockManager.java:218) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.locks.impl.InfinispanLock$LockPlaceHolder.checkState(InfinispanLock.java:436) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.locks.impl.InfinispanLock$LockPlaceHolder.toInvocationStage(InfinispanLock.java:408) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.toInvocationStage(DefaultLockManager.java:248) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.toInvocationStage(DefaultLockManager.java:272) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.nonTxLockAndInvokeNext(AbstractLockingInterceptor.java:294) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:126) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitDataWriteCommand(NonTransactionalLockingInterceptor.java:40) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitRemoveCommand(AbstractLockingInterceptor.java:102) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:66) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:55) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:306) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:252) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.statetransfer.StateTransferInterceptor.visitRemoveCommand(StateTransferInterceptor.java:108) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:66) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:55) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitRemoveCommand(CacheMgmtInterceptor.java:427) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:66) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:55) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.DDAsyncInterceptor.visitRemoveCommand(DDAsyncInterceptor.java:65) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:66) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:55) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:123) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:56) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.DDAsyncInterceptor.visitRemoveCommand(DDAsyncInterceptor.java:65) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:66) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:55) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:50) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:234) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.remote.BaseRpcInvokingCommand.processVisitableCommandAsync(BaseRpcInvokingCommand.java:63) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.remote.SingleRpcCommand.invokeAsync(SingleRpcCommand.java:57) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokeCommand(BasePerCacheInboundInvocationHandler.java:94) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.invoke(BaseBlockingRunnable.java:99) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.runAsync(BaseBlockingRunnable.java:71) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:40) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_162] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_162] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_162] 2018-08-22 11:45:16,555 WARN [org.infinispan.remoting.inboundhandler.TrianglePerCacheInboundInvocationHandler] (remote-thread--p2-t1:[]) ISPN000071: Caught exception when handling command SingleRpcCommand{cacheName='weather', command=RemoveExpiredCommand{key=Brazil, value=LocationWeather{temperature=37.0, country='Brazil'}, lifespan=null, maxIde=true}} org.infinispan.util.concurrent.TimeoutException: ISPN000299: Unable to acquire lock after 10 seconds for key Brazil and requestor CommandInvocation:Diegos-MacBook-Pro-45456:1. Lock is held by CommandInvocation:Diegos-MacBook-Pro-29329:1 at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.get(DefaultLockManager.java:288) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.get(DefaultLockManager.java:218) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.locks.impl.InfinispanLock$LockPlaceHolder.checkState(InfinispanLock.java:436) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.locks.impl.InfinispanLock$LockPlaceHolder.toInvocationStage(InfinispanLock.java:408) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.toInvocationStage(DefaultLockManager.java:248) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.toInvocationStage(DefaultLockManager.java:272) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.nonTxLockAndInvokeNext(AbstractLockingInterceptor.java:294) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:126) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitDataWriteCommand(NonTransactionalLockingInterceptor.java:40) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitRemoveCommand(AbstractLockingInterceptor.java:102) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:66) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:55) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:306) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:252) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.statetransfer.StateTransferInterceptor.visitRemoveCommand(StateTransferInterceptor.java:108) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:66) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:55) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitRemoveCommand(CacheMgmtInterceptor.java:427) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:66) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:55) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.DDAsyncInterceptor.visitRemoveCommand(DDAsyncInterceptor.java:65) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:66) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:55) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:123) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:56) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.DDAsyncInterceptor.visitRemoveCommand(DDAsyncInterceptor.java:65) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.Visitor.visitRemoveExpiredCommand(Visitor.java:66) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.write.RemoveExpiredCommand.acceptVisitor(RemoveExpiredCommand.java:55) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:50) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:234) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.remote.BaseRpcInvokingCommand.processVisitableCommandAsync(BaseRpcInvokingCommand.java:63) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.commands.remote.SingleRpcCommand.invokeAsync(SingleRpcCommand.java:57) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokeCommand(BasePerCacheInboundInvocationHandler.java:94) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.invoke(BaseBlockingRunnable.java:99) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.runAsync(BaseBlockingRunnable.java:71) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:40) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_162] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_162] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_162] 2018-08-22 11:45:21,546 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (timeout-thread--p3-t1:[]) ISPN000136: Error executing command RemoveExpiredCommand, writing keys [Brazil] org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks. Id=1 at org.infinispan.util.concurrent.CommandAckCollector.createTimeoutException(CommandAckCollector.java:222) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.CommandAckCollector.access$800(CommandAckCollector.java:58) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:275) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:255) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_162] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_162] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [?:1.8.0_162] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_162] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_162] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_162] 2018-08-22 11:45:21,547 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (main:[]) ISPN000136: Error executing command GetKeyValueCommand, writing keys [] org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks. Id=1 at org.infinispan.util.concurrent.CommandAckCollector.createTimeoutException(CommandAckCollector.java:222) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.CommandAckCollector.access$800(CommandAckCollector.java:58) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:275) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:255) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_162] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_162] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_162] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_162] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_162] at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_162] 2018-08-22 11:45:21,547 WARN [org.infinispan.expiration.impl.ClusterExpirationManager] (expiration-thread--p6-t1:[]) ISPN000026: Caught exception purging data container! java.util.concurrent.CompletionException: org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks. Id=1 at java.util.concurrent.CompletableFuture.reportJoin(CompletableFuture.java:375) ~[?:1.8.0_162] at java.util.concurrent.CompletableFuture.join(CompletableFuture.java:1934) ~[?:1.8.0_162] at java.util.ArrayList.forEach(ArrayList.java:1257) ~[?:1.8.0_162] at org.infinispan.expiration.impl.ClusterExpirationManager.processExpiration(ClusterExpirationManager.java:104) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.expiration.impl.ExpirationManagerImpl$ScheduledTask.run(ExpirationManagerImpl.java:243) [infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_162] at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [?:1.8.0_162] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_162] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [?:1.8.0_162] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_162] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_162] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_162] Caused by: org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks. Id=1 at org.infinispan.util.concurrent.CommandAckCollector.createTimeoutException(CommandAckCollector.java:222) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.CommandAckCollector.access$800(CommandAckCollector.java:58) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:275) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:255) ~[infinispan-embedded-9.4.0-SNAPSHOT.jar:9.4.0-SNAPSHOT] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_162] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_162] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_162] ... 3 more Exception in thread "main" org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks. Id=1 at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:259) at org.infinispan.cache.impl.CacheImpl.get(CacheImpl.java:479) at org.infinispan.cache.impl.DecoratedCache.get(DecoratedCache.java:529) at org.infinispan.cache.impl.AbstractDelegatingCache.get(AbstractDelegatingCache.java:348) at org.infinispan.cache.impl.EncoderCache.get(EncoderCache.java:659) at com.github.diegolovison.example.infinispan.ClusteredCacheManagerExample.main(ClusteredCacheManagerExample.java:50) Caused by: org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks. Id=1 at org.infinispan.util.concurrent.CommandAckCollector.createTimeoutException(CommandAckCollector.java:222) at org.infinispan.util.concurrent.CommandAckCollector.access$800(CommandAckCollector.java:58) at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:275) at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:255) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Suppressed: org.infinispan.util.logging.TraceException at org.infinispan.interceptors.impl.SimpleAsyncInvocationStage.get(SimpleAsyncInvocationStage.java:41) at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:250) at org.infinispan.cache.impl.CacheImpl.get(CacheImpl.java:479) at org.infinispan.cache.impl.DecoratedCache.get(DecoratedCache.java:529) at org.infinispan.cache.impl.AbstractDelegatingCache.get(AbstractDelegatingCache.java:348) at org.infinispan.cache.impl.EncoderCache.get(EncoderCache.java:659) at com.github.diegolovison.example.infinispan.ClusteredCacheManagerExample.main(ClusteredCacheManagerExample.java:50) {noformat} {code:java} public class ClusteredCacheManagerExample { private static final String DEFAULT_LOCATION = "Brazil"; public static void main(String[] args) throws InterruptedException { System.out.println(getPID()); EmbeddedCacheManager cacheManager = new DefaultCacheManager(GlobalConfigurationBuilder.defaultClusteredBuilder() .transport().clusterName("WeatherApp") .addProperty("configurationFile", "default-jgroups-tcp.xml") .build()); cacheManager.defineConfiguration("weather", new ConfigurationBuilder() .clustering() .cacheMode(CacheMode.DIST_SYNC) .hash().numOwners(2) .expiration().maxIdle(15, TimeUnit.SECONDS) .build()); cacheManager.addListener(new ClusterListener()); Cache<String, LocationWeather> cache = cacheManager.getCache("weather"); cache.addListener(new CacheListener()); if (cacheManager.isCoordinator()) { cache.put(DEFAULT_LOCATION, new LocationWeather(37, DEFAULT_LOCATION)); } AdvancedCache<String, LocationWeather> localCache = cache.getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL); while (true) { System.out.println(new Date() + ": " + localCache.get(DEFAULT_LOCATION)); TimeUnit.SECONDS.sleep(5); } } public static long getPID() { String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName(); if (processName != null && processName.length() > 0) { try { return Long.parseLong(processName.split("@")[0]); } catch (Exception e) { return 0; } } return 0; } } @Listener public class ClusterListener { @ViewChanged public void viewChanged(ViewChangedEvent event) { System.out.println(event); } } @Listener(clustered = true) public class CacheListener { @CacheEntryCreated public void entryCreated(CacheEntryCreatedEvent<String, LocationWeather> event) { if (!event.isOriginLocal()) { System.out.printf("-- Entry for %s modified by another node in the cluster\n", event.getKey()); } } } public class LocationWeather implements Serializable { private final float temperature; private final String country; public LocationWeather(float temperature, String country) { this.temperature = temperature; this.country = country; } @Override public String toString() { return "LocationWeather{" + "temperature=" + temperature + ", country='" + country + '\'' + '}'; } } {code} If I coment the line *.expiration().maxIdle(15, TimeUnit.SECONDS)* it starts woking.