TwoWaySplitAndMerge[DIST_SYNC] deadlock in conflict resolution -------------------------------------------------------------- Key: ISPN-8538 URL: https://issues.jboss.org/browse/ISPN-8538 Project: Infinispan Issue Type: Bug Components: Core, Test Suite - Core Affects Versions: 9.2.0.Beta1 Reporter: Dan Berindei Assignee: Ryan Emerson Labels: testsuite_stability Fix For: 9.2.0.Beta2 Attachments: TwoWaySplitAndMergeTest_ISPN-factory_problems_20171115.log.gz, stack_20171115_1505.txt This might be related to me having trace logging enabled. Main thread is stuck in Main thread: {noformat} "testng-TwoWaySplitAndMergeTest[DIST_SYNC]" #27 prio=5 os_prio=0 tid=0x00007f52a4faa000 nid=0x3f93 waiting for monitor entry [0x00007f51feaf2000] java.lang.Thread.State: BLOCKED (on object monitor) at org.infinispan.topology.ClusterCacheStatus.doLeave(ClusterCacheStatus.java:723) - waiting to lock <0x00000000e0d4c1a8> (a org.infinispan.topology.ClusterCacheStatus) at org.infinispan.topology.ClusterTopologyManagerImpl.handleLeave(ClusterTopologyManagerImpl.java:250) at org.infinispan.topology.CacheTopologyControlCommand.doPerform(CacheTopologyControlCommand.java:186) at org.infinispan.topology.CacheTopologyControlCommand.invokeAsync(CacheTopologyControlCommand.java:166) at org.infinispan.commands.ReplicableCommand.invoke(ReplicableCommand.java:44) at org.infinispan.topology.LocalTopologyManagerImpl.executeOnCoordinator(LocalTopologyManagerImpl.java:692) at org.infinispan.topology.LocalTopologyManagerImpl.leave(LocalTopologyManagerImpl.java:197) at org.infinispan.statetransfer.StateTransferManagerImpl.stop(StateTransferManagerImpl.java:264) at sun.reflect.GeneratedMethodAccessor222.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.infinispan.commons.util.SecurityActions.lambda$invokeAccessibly$0(SecurityActions.java:91) at org.infinispan.commons.util.SecurityActions$$Lambda$159/273460605.run(Unknown Source) at org.infinispan.commons.util.SecurityActions.doPrivileged(SecurityActions.java:83) at org.infinispan.commons.util.SecurityActions.invokeAccessibly(SecurityActions.java:88) at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:165) at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:883) at org.infinispan.factories.AbstractComponentRegistry.internalStop(AbstractComponentRegistry.java:684) at org.infinispan.factories.AbstractComponentRegistry.stop(AbstractComponentRegistry.java:583) - locked <0x00000000c83a6ad8> (a org.infinispan.factories.ComponentRegistry) at org.infinispan.factories.ComponentRegistry.stop(ComponentRegistry.java:259) at org.infinispan.cache.impl.CacheImpl.performImmediateShutdown(CacheImpl.java:1046) at org.infinispan.cache.impl.CacheImpl.stop(CacheImpl.java:1010) at org.infinispan.cache.impl.AbstractDelegatingCache.stop(AbstractDelegatingCache.java:420) at org.infinispan.manager.DefaultCacheManager.terminate(DefaultCacheManager.java:687) at org.infinispan.manager.DefaultCacheManager.stopCaches(DefaultCacheManager.java:727) at org.infinispan.manager.DefaultCacheManager.stop(DefaultCacheManager.java:704) at org.infinispan.test.TestingUtil.killCacheManagers(TestingUtil.java:774) at org.infinispan.test.MultipleCacheManagersTest.clearContent(MultipleCacheManagersTest.java:146) Locked ownable synchronizers: - <0x00000000c464a9d8> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00000000e12f5a60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) {noformat} Two other threads are involved: {noformat} "transport-thread-TwoWaySplitAndMergeTest[DIST_SYNC]-NodeA-p26871-t1" #93288 daemon prio=5 os_prio=0 tid=0x00007f51cc03d800 nid=0x4ab0 waiting for monitor entry [0x00007f515dd70000] java.lang.Thread.State: BLOCKED (on object monitor) at org.infinispan.conflict.impl.StateReceiverImpl$SegmentRequest.cancel(StateReceiverImpl.java:213) - waiting to lock <0x00000000c7476a38> (a org.infinispan.conflict.impl.StateReceiverImpl$SegmentRequest) at org.infinispan.conflict.impl.StateReceiverImpl.stop(StateReceiverImpl.java:76) - locked <0x00000000e0d60fe8> (a org.infinispan.conflict.impl.StateReceiverImpl) at org.infinispan.conflict.impl.DefaultConflictManager$ReplicaSpliterator.tryAdvance(DefaultConflictManager.java:463) at java.util.Spliterator.forEachRemaining(Spliterator.java:326) at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471) at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151) at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174) at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418) at org.infinispan.conflict.impl.DefaultConflictManager.doResolveConflicts(DefaultConflictManager.java:265) at org.infinispan.conflict.impl.DefaultConflictManager.resolveConflicts(DefaultConflictManager.java:252) at org.infinispan.topology.ClusterCacheStatus.updateTopologiesAfterMerge(ClusterCacheStatus.java:193) - locked <0x00000000e0d4c1a8> (a org.infinispan.topology.ClusterCacheStatus) at org.infinispan.partitionhandling.impl.PreferAvailabilityStrategy.onPartitionMerge(PreferAvailabilityStrategy.java:197) at org.infinispan.topology.ClusterCacheStatus.doMergePartitions(ClusterCacheStatus.java:597) - locked <0x00000000e0d4c1a8> (a org.infinispan.topology.ClusterCacheStatus) at org.infinispan.topology.ClusterTopologyManagerImpl.lambda$recoverClusterStatus$6(ClusterTopologyManagerImpl.java:528) at org.infinispan.topology.ClusterTopologyManagerImpl$$Lambda$753/1411554646.run(Unknown Source) at org.infinispan.executors.LimitedExecutor.runTasks(LimitedExecutor.java:144) at org.infinispan.executors.LimitedExecutor.access$100(LimitedExecutor.java:33) at org.infinispan.executors.LimitedExecutor$Runner.run(LimitedExecutor.java:174) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000000cef073e8> (a java.util.concurrent.ThreadPoolExecutor$Worker) {noformat} {noformat} "remote-thread-TwoWaySplitAndMergeTest[DIST_SYNC]-NodeA-p26869-t6" #93412 daemon prio=5 os_prio=0 tid=0x00007f5238071000 nid=0x4b3c waiting for monitor entry [0x00007f51870ee000] java.lang.Thread.State: BLOCKED (on object monitor) at org.infinispan.conflict.impl.StateReceiverImpl$SegmentRequest.clear(StateReceiverImpl.java:182) - waiting to lock <0x00000000e0d60fe8> (a org.infinispan.conflict.impl.StateReceiverImpl) - locked <0x00000000c7476a38> (a org.infinispan.conflict.impl.StateReceiverImpl$SegmentRequest) at org.infinispan.conflict.impl.StateReceiverImpl$SegmentRequest.cancel(StateReceiverImpl.java:220) - locked <0x00000000c7476a38> (a org.infinispan.conflict.impl.StateReceiverImpl$SegmentRequest) at org.infinispan.conflict.impl.StateReceiverImpl$SegmentRequest.lambda$requestState$1(StateReceiverImpl.java:164) at org.infinispan.conflict.impl.StateReceiverImpl$SegmentRequest$$Lambda$1997/1053695950.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniExceptionally(CompletableFuture.java:870) at java.util.concurrent.CompletableFuture$UniExceptionally.tryFire(CompletableFuture.java:852) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) at org.infinispan.statetransfer.InboundTransferTask.notifyCompletion(InboundTransferTask.java:245) at org.infinispan.statetransfer.InboundTransferTask.onStateReceived(InboundTransferTask.java:238) at org.infinispan.conflict.impl.StateReceiverImpl$SegmentRequest.receiveState(StateReceiverImpl.java:205) - locked <0x00000000c7476a38> (a org.infinispan.conflict.impl.StateReceiverImpl$SegmentRequest) at org.infinispan.conflict.impl.StateReceiverImpl.receiveState(StateReceiverImpl.java:108) at org.infinispan.statetransfer.StateResponseCommand.invokeAsync(StateResponseCommand.java:90) at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokeCommand(BasePerCacheInboundInvocationHandler.java:102) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.invoke(BaseBlockingRunnable.java:99) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.runAsync(BaseBlockingRunnable.java:71) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:40) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000000cf105ce8> (a java.util.concurrent.ThreadPoolExecutor$Worker) {noformat}