Staggered remote get throws IllegalStateException ------------------------------------------------- Key: ISPN-9099 URL: https://issues.jboss.org/browse/ISPN-9099 Project: Infinispan Issue Type: Bug Components: Core Affects Versions: 9.2.1.Final Reporter: Dan Berindei Assignee: Dan Berindei Labels: testsuite_stability Fix For: 9.3.0.Beta1 {{StaggeredRequest.sendNextMessage()}} throws {{IllegalStateException}} if the last target has left the cluster and the next-to-last target didn't reply within the stagger timeout. This causes a random failure in {{TwoWaySplitAndMergeTest}}: {noformat} 12:29:10,960 ERROR (testng-TwoWaySplitAndMergeTest[DIST_SYNC]:[]) [TestSuiteProgress] Test failed: org.infinispan.partitionhandling.TwoWaySplitAndMergeTest.testSplitAndMerge4[DIST_SYNC] org.infinispan.commons.CacheException: java.lang.IllegalStateException: Request should have been completed already. at org.infinispan.interceptors.impl.InvocationContextInterceptor.rethrowException(InvocationContextInterceptor.java:134) ~[classes/:?] at org.infinispan.interceptors.impl.InvocationContextInterceptor.lambda$new$0(InvocationContextInterceptor.java:62) ~[classes/:?] at org.infinispan.interceptors.InvocationExceptionFunction.apply(InvocationExceptionFunction.java:21) ~[classes/:?] at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.invokeQueuedHandlers(QueueAsyncInvocationStage.java:118) ~[classes/:?] at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.accept(QueueAsyncInvocationStage.java:81) ~[classes/:?] at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.accept(QueueAsyncInvocationStage.java:30) ~[classes/:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760) ~[?:1.8.0_171] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736) ~[?:1.8.0_171] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_171] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) ~[?:1.8.0_171] at org.infinispan.remoting.transport.AbstractRequest.completeExceptionally(AbstractRequest.java:74) ~[classes/:?] at org.infinispan.remoting.transport.jgroups.StaggeredRequest.sendNextMessage(StaggeredRequest.java:106) ~[classes/:?] at org.infinispan.remoting.transport.jgroups.StaggeredRequest.onTimeout(StaggeredRequest.java:66) ~[classes/:?] at org.infinispan.remoting.transport.AbstractRequest.call(AbstractRequest.java:87) ~[classes/:?] at org.infinispan.remoting.transport.AbstractRequest.call(AbstractRequest.java:22) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_171] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_171] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_171] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_171] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_171] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_171] Suppressed: java.util.concurrent.ExecutionException: org.infinispan.commons.CacheException: java.lang.IllegalStateException: Request should have been completed already. at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357) ~[?:1.8.0_171] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1915) ~[?:1.8.0_171] at org.infinispan.util.concurrent.CompletableFutures.await(CompletableFutures.java:82) ~[classes/:?] at org.infinispan.interceptors.impl.SimpleAsyncInvocationStage.get(SimpleAsyncInvocationStage.java:37) ~[classes/:?] at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:250) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.get(CacheImpl.java:485) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.get(CacheImpl.java:478) ~[classes/:?] at org.infinispan.cache.impl.AbstractDelegatingCache.get(AbstractDelegatingCache.java:348) ~[classes/:?] at org.infinispan.cache.impl.EncoderCache.get(EncoderCache.java:658) ~[classes/:?] at org.infinispan.partitionhandling.BasePartitionHandlingTest.assertKeyAvailableForRead(BasePartitionHandlingTest.java:396) ~[test-classes/:?] at org.infinispan.partitionhandling.BasePartitionHandlingTest$Partition.assertKeyAvailableForRead(BasePartitionHandlingTest.java:325) ~[test-classes/:?] at org.infinispan.partitionhandling.TwoWaySplitAndMergeTest.lambda$testSplitAndMerge$1(TwoWaySplitAndMergeTest.java:96) ~[test-classes/:?] at java.util.stream.Streams$RangeIntSpliterator.forEachRemaining(Streams.java:110) ~[?:1.8.0_171] at java.util.stream.IntPipeline$Head.forEach(IntPipeline.java:557) ~[?:1.8.0_171] at org.infinispan.partitionhandling.TwoWaySplitAndMergeTest.testSplitAndMerge(TwoWaySplitAndMergeTest.java:95) ~[test-classes/:?] at org.infinispan.partitionhandling.TwoWaySplitAndMergeTest.testSplitAndMerge4(TwoWaySplitAndMergeTest.java:43) ~[test-classes/:?] Caused by: org.infinispan.commons.CacheException: java.lang.IllegalStateException: Request should have been completed already. at org.infinispan.interceptors.impl.InvocationContextInterceptor.rethrowException(InvocationContextInterceptor.java:134) ~[classes/:?] at org.infinispan.interceptors.impl.InvocationContextInterceptor.lambda$new$0(InvocationContextInterceptor.java:62) ~[classes/:?] at org.infinispan.interceptors.InvocationExceptionFunction.apply(InvocationExceptionFunction.java:21) ~[classes/:?] at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.invokeQueuedHandlers(QueueAsyncInvocationStage.java:118) ~[classes/:?] at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.accept(QueueAsyncInvocationStage.java:81) ~[classes/:?] at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.accept(QueueAsyncInvocationStage.java:30) ~[classes/:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760) ~[?:1.8.0_171] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736) ~[?:1.8.0_171] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_171] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) ~[?:1.8.0_171] at org.infinispan.remoting.transport.AbstractRequest.completeExceptionally(AbstractRequest.java:74) ~[classes/:?] at org.infinispan.remoting.transport.jgroups.StaggeredRequest.sendNextMessage(StaggeredRequest.java:106) ~[classes/:?] at org.infinispan.remoting.transport.jgroups.StaggeredRequest.onTimeout(StaggeredRequest.java:66) ~[classes/:?] at org.infinispan.remoting.transport.AbstractRequest.call(AbstractRequest.java:87) ~[classes/:?] at org.infinispan.remoting.transport.AbstractRequest.call(AbstractRequest.java:22) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_171] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_171] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_171] ... 3 more Caused by: java.lang.IllegalStateException: Request should have been completed already. at org.infinispan.remoting.transport.jgroups.StaggeredRequest.sendNextMessage(StaggeredRequest.java:88) ~[classes/:?] ... 9 more {noformat}