[infinispan-issues] [Red Hat JIRA] (ISPN-12277) Testcontainers ResourceReaper::start blocks test startup

Testcontainers ResourceReaper::start blocks test startup -------------------------------------------------------- Key: ISPN-12277 URL: https://issues.redhat.com/browse/ISPN-12277 Project: Infinispan Issue Type: Bug Components: Test Suite Affects Versions: 12.0.0.Dev02 Reporter: Diego Lovison Assignee: Diego Lovison Priority: Major Fix For: 12.1.0.Final The current Testcontainers version 1.14.0 has the Thread.State: BLOCKED Upgrading to 1.14.3 ( the latest version at this time ) won't fix the issue. It is missing a timeout in the socket that connects with Ryuk Upstream issue: https://github.com/testcontainers/testcontainers-java/issues/3183 It is affecting different tests {noformat} 2020-09-01 07:57:17'. Full thread dump OpenJDK 64-Bit Server VM (11.0.7+10 mixed mode):'. Threads class SMR info:'. _java_thread_list=0x00007f6e980af4b0, length=65, elements={'. 0x00007f6ee0018000, 0x00007f6ee0193800, 0x00007f6ee0198000, 0x00007f6ee01b0800,'. 0x00007f6ee01b7000, 0x00007f6ee01b9000, 0x00007f6ee022f000, 0x00007f6ee0294000,'. 0x00007f6ee034c800, 0x00007f6ee15fc800, 0x00007f6ee194b800, 0x00007f6ee1956000,'. 0x00007f6ee1ed8800, 0x00007f6ee2097000, 0x00007f6ee1f2e800, 0x00007f6ee1f1e000,'. 0x00007f6e9400c000, 0x00007f6e6410f800, 0x00007f6eb0008000, 0x00007f6ee1ff9800,'. 0x00007f6ee279c000, 0x00007f6ee2697000, 0x00007f6ee23ec800, 0x00007f6ee2675800,'. 0x00007f6ee23f8000, 0x00007f6ee2686000, 0x00007f6eac873000, 0x00007f6eac875800,'. 0x00007f6ee2674800, 0x00007f6ebc01d000, 0x00007f6ee270a800, 0x00007f6ee23e6800,'. 0x00007f6ebc003000, 0x00007f6ee27ec800, 0x00007f6e70024000, 0x00007f6ee2684800,'. 0x00007f6ee270c000, 0x00007f6ee26e1800, 0x00007f6ee23b8000, 0x00007f6ee2721800,'. 0x00007f6e98089800, 0x00007f6e74062800, 0x00007f6ea00aa800, 0x00007f6ea00b8000,'. 0x00007f6e70026000, 0x00007f6ee2720000, 0x00007f6ee23ed800, 0x00007f6ee27b5800,'. 0x00007f6ee27b6000, 0x00007f6ee278c000, 0x00007f6ea8ba8800, 0x00007f6ee278c800,'. 0x00007f6eac876800, 0x00007f6ee2707000, 0x00007f6ee2708000, 0x00007f6eac877800,'. 0x00007f6ee2708800, 0x00007f6e684fb800, 0x00007f6e9c02e000, 0x00007f6e7000c000,'. 0x00007f6ee27e0000, 0x00007f6e8cb8c000, 0x00007f6e7c223800, 0x00007f6e641e6000,'. 0x00007f6e641e0000'. }'. "testng-AuthenticationKeyCloakIT" #1 prio=5 os_prio=0 cpu=74322.16ms elapsed=50777.64s tid=0x00007f6ee0018000 nid=0x466d waiting for monitor entry [ java.lang.Thread.State: BLOCKED (on object monitor)'. at org.testcontainers.utility.ResourceReaper.start(ResourceReaper.java:108)'. - waiting to lock <0x00000000c245cd18> (a java.util.ArrayList)'. at org.testcontainers.DockerClientFactory.client(DockerClientFactory.java:155)'. - locked <0x00000000c2215790> (a [Ljava.lang.Object;)'. at org.testcontainers.LazyDockerClient.getDockerClient(LazyDockerClient.java:14)'. at org.testcontainers.LazyDockerClient.listImagesCmd(LazyDockerClient.java:12)'. at org.testcontainers.images.LocalImagesCache.maybeInitCache(LocalImagesCache.java:68)'. - locked <0x00000000d0f8de98> (a org.testcontainers.images.LocalImagesCache)'. at org.testcontainers.images.LocalImagesCache.get(LocalImagesCache.java:32)'. at org.testcontainers.images.AbstractImagePullPolicy.shouldPull(AbstractImagePullPolicy.java:18)'. at org.testcontainers.images.RemoteDockerImage.resolve(RemoteDockerImage.java:59)'. at org.testcontainers.images.RemoteDockerImage.resolve(RemoteDockerImage.java:26)'. at org.testcontainers.utility.LazyFuture.getResolvedValue(LazyFuture.java:20)'. - locked <0x00000000d0f8e090> (a java.util.concurrent.atomic.AtomicReference)'. at org.testcontainers.utility.LazyFuture.get(LazyFuture.java:27)'. at org.testcontainers.containers.GenericContainer.getDockerImageName(GenericContainer.java:1266)'. at org.testcontainers.containers.GenericContainer.logger(GenericContainer.java:603)'. at org.testcontainers.containers.GenericContainer.doStart(GenericContainer.java:309)'. at org.testcontainers.containers.GenericContainer.start(GenericContainer.java:300)'. at org.infinispan.server.test.core.KeyCloakServerRule.before(KeyCloakServerRule.java:79)'. at org.infinispan.server.test.core.KeyCloakServerRule.access$000(KeyCloakServerRule.java:33)'. at org.infinispan.server.test.core.KeyCloakServerRule$1.evaluate(KeyCloakServerRule.java:48)'. at org.infinispan.server.test.junit4.InfinispanServerRule$1.evaluate(InfinispanServerRule.java:77)'. at org.junit.rules.RunRules.evaluate(RunRules.java:20)'. at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)'. at org.junit.runners.ParentRunner.run(ParentRunner.java:413)'. at org.junit.runners.Suite.runChild(Suite.java:128)'. at org.junit.runners.Suite.runChild(Suite.java:27)'. at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)'. at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)'. at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)'. at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)'. at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)'. at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)'. at org.junit.runners.ParentRunner.run(ParentRunner.java:413)'. at org.apache.maven.surefire.junitcore.JUnitCore.run(JUnitCore.java:55)'. at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:137)'. at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:107)'. at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:83)'. at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:75)'. at org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:158)'. at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:377)'. at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:138)'. at org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:465)'. at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:451)'. "Reference Handler" #2 daemon prio=10 os_prio=0 cpu=31.80ms elapsed=50777.60s tid=0x00007f6ee0193800 nid=0x4674 waiting on condition [ java.lang.Thread.State: RUNNABLE'. at java.lang.ref.Reference.waitForReferencePendingList(java.base(a)11.0.7/Native Method)'. at java.lang.ref.Reference.processPendingReferences(java.base@11.0.7/Reference.java:241)'. at java.lang.ref.Reference$ReferenceHandler.run(java.base@11.0.7/Reference.java:213)'. "Finalizer" #3 daemon prio=8 os_prio=0 cpu=39.97ms elapsed=50777.60s tid=0x00007f6ee0198000 nid=0x4675 in Object.wait() [0x00007f6ecc6ee000]'. java.lang.Thread.State: WAITING (on object monitor)'. at java.lang.Object.wait(java.base(a)11.0.7/Native Method)'. - waiting on <0x00000000c0a17250> (a java.lang.ref.ReferenceQueue$Lock)'. at java.lang.ref.ReferenceQueue.remove(java.base@11.0.7/ReferenceQueue.java:155)'. - waiting to re-lock in wait() <0x00000000c0a17250> (a java.lang.ref.ReferenceQueue$Lock)'. at java.lang.ref.ReferenceQueue.remove(java.base@11.0.7/ReferenceQueue.java:176)'. at java.lang.ref.Finalizer$FinalizerThread.run(java.base@11.0.7/Finalizer.java:170)'. "Signal Dispatcher" #4 daemon prio=9 os_prio=0 cpu=0.35ms elapsed=50777.59s tid=0x00007f6ee01b0800 nid=0x4676 waiting on condition [ java.lang.Thread.State: RUNNABLE'. "C2 CompilerThread0" #5 daemon prio=9 os_prio=0 cpu=44410.36ms elapsed=50777.59s tid=0x00007f6ee01b7000 nid=0x4677 waiting on condition [ java.lang.Thread.State: RUNNABLE'. No compile task'. "Sweeper thread" #7 daemon prio=9 os_prio=0 cpu=2201.68ms elapsed=50777.59s tid=0x00007f6ee01b9000 nid=0x4678 runnable [0x0000000000000000]'. java.lang.Thread.State: RUNNABLE'. "Common-Cleaner" #8 daemon prio=8 os_prio=0 cpu=106.13ms elapsed=50777.55s tid=0x00007f6ee022f000 nid=0x4679 in Object.wait() [0x00007f6ecc2ea000]'. java.lang.Thread.State: TIMED_WAITING (on object monitor)'. at java.lang.Object.wait(java.base(a)11.0.7/Native Method)'. - waiting on <0x00000000c0475478> (a java.lang.ref.ReferenceQueue$Lock)'. at java.lang.ref.ReferenceQueue.remove(java.base@11.0.7/ReferenceQueue.java:155)'. - waiting to re-lock in wait() <0x00000000c0475478> (a java.lang.ref.ReferenceQueue$Lock)'. at jdk.internal.ref.CleanerImpl.run(java.base@11.0.7/CleanerImpl.java:148)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. at jdk.internal.misc.InnocuousThread.run(java.base@11.0.7/InnocuousThread.java:134)'. "Service Thread" #9 daemon prio=9 os_prio=0 cpu=0.20ms elapsed=50777.50s tid=0x00007f6ee0294000 nid=0x467a runnable [0x0000000000000000]'. java.lang.Thread.State: RUNNABLE'. "surefire-forkedjvm-command-thread" #10 daemon prio=5 os_prio=0 cpu=418.75ms elapsed=50777.42s tid=0x00007f6ee034c800 nid=0x467d runnable [ java.lang.Thread.State: RUNNABLE'. at java.io.FileInputStream.readBytes(java.base(a)11.0.7/Native Method)'. at java.io.FileInputStream.read(java.base@11.0.7/FileInputStream.java:279)'. at java.io.BufferedInputStream.fill(java.base@11.0.7/BufferedInputStream.java:252)'. at java.io.BufferedInputStream.read(java.base@11.0.7/BufferedInputStream.java:271)'. - locked <0x00000000c0a1ebd0> (a java.io.BufferedInputStream)'. at java.io.DataInputStream.readInt(java.base@11.0.7/DataInputStream.java:392)'. at org.apache.maven.surefire.booter.MasterProcessCommand.decode(MasterProcessCommand.java:113)'. at org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:383)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "mysql-cj-abandoned-connection-cleanup" #13 daemon prio=5 os_prio=0 cpu=2511.81ms elapsed=50770.55s tid=0x00007f6ee15fc800 nid=0x4681 in java.lang.Thread.State: TIMED_WAITING (on object monitor)'. at java.lang.Object.wait(java.base(a)11.0.7/Native Method)'. - waiting on <0x00000000c10f88f0> (a java.lang.ref.ReferenceQueue$Lock)'. at java.lang.ref.ReferenceQueue.remove(java.base@11.0.7/ReferenceQueue.java:155)'. - waiting to re-lock in wait() <0x00000000c10f88f0> (a java.lang.ref.ReferenceQueue$Lock)'. at com.mysql.cj.jdbc.AbandonedConnectionCleanupThread.run(AbandonedConnectionCleanupThread.java:85)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1128)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "RxSchedulerPurge-1" #14 daemon prio=5 os_prio=0 cpu=4517.04ms elapsed=50769.59s tid=0x00007f6ee194b800 nid=0x4682 waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000c1370338> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.7/AbstractQueuedSynchronizer.java:2123)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1182)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "RxCachedWorkerPoolEvictor-1" #15 daemon prio=5 os_prio=0 cpu=142.40ms elapsed=50769.59s tid=0x00007f6ee1956000 nid=0x4683 waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000c1372c40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.7/AbstractQueuedSynchronizer.java:2123)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1182)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "ForkJoinPool.commonPool-worker-3" #34 daemon prio=5 os_prio=0 cpu=80.38ms elapsed=50765.05s tid=0x00007f6ee1ed8800 nid=0x4699 waiting on condition java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000c0fdda88> (a java.util.concurrent.ForkJoinPool)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.ForkJoinPool.runWorker(java.base@11.0.7/ForkJoinPool.java:1628)'. at java.util.concurrent.ForkJoinWorkerThread.run(java.base@11.0.7/ForkJoinWorkerThread.java:177)'. "RunningTestsRegistry-Worker" #40 prio=5 os_prio=0 cpu=13.12ms elapsed=50764.58s tid=0x00007f6ee2097000 nid=0x469f waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000c1eb3768> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1170)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "testcontainers-ryuk" #70 daemon prio=5 os_prio=0 cpu=588.25ms elapsed=50762.21s tid=0x00007f6ee1f2e800 nid=0x473c runnable [0x00007f6e88167000]'. java.lang.Thread.State: RUNNABLE'. at java.net.SocketInputStream.socketRead0(java.base(a)11.0.7/Native Method)'. at java.net.SocketInputStream.socketRead(java.base@11.0.7/SocketInputStream.java:115)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:168)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:140)'. at sun.nio.cs.StreamDecoder.readBytes(java.base@11.0.7/StreamDecoder.java:284)'. at sun.nio.cs.StreamDecoder.implRead(java.base@11.0.7/StreamDecoder.java:326)'. at sun.nio.cs.StreamDecoder.read(java.base@11.0.7/StreamDecoder.java:178)'. - locked <0x00000000d495a2c0> (a java.io.InputStreamReader)'. at java.io.InputStreamReader.read(java.base@11.0.7/InputStreamReader.java:185)'. at java.io.BufferedReader.fill(java.base@11.0.7/BufferedReader.java:161)'. at java.io.BufferedReader.readLine(java.base@11.0.7/BufferedReader.java:326)'. - locked <0x00000000d495a2c0> (a java.io.InputStreamReader)'. at java.io.BufferedReader.readLine(java.base@11.0.7/BufferedReader.java:392)'. at org.testcontainers.utility.ResourceReaper$FilterRegistry.waitForAcknowledgment(ResourceReaper.java:416)'. at org.testcontainers.utility.ResourceReaper$FilterRegistry.register(ResourceReaper.java:410)'. at org.testcontainers.utility.ResourceReaper.lambda$null$1(ResourceReaper.java:135)'. - locked <0x00000000c245cd18> (a java.util.ArrayList)'. at org.testcontainers.utility.ResourceReaper$$Lambda$1097/0x000000010099b040.run(Unknown Source)'. at org.rnorth.ducttape.ratelimits.RateLimiter.doWhenReady(RateLimiter.java:27)'. at org.testcontainers.utility.ResourceReaper.lambda$start$2(ResourceReaper.java:119)'. at org.testcontainers.utility.ResourceReaper$$Lambda$1096/0x000000010099bc40.run(Unknown Source)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "testcontainers-ryuk" #71 daemon prio=5 os_prio=0 cpu=156913.42ms elapsed=50731.69s tid=0x00007f6ee1f1e000 nid=0x478e waiting on condition [ java.lang.Thread.State: TIMED_WAITING (sleeping)'. at java.lang.Thread.sleep(java.base(a)11.0.7/Native Method)'. at org.rnorth.ducttape.ratelimits.RateLimiter.doWhenReady(RateLimiter.java:23)'. at org.testcontainers.utility.ResourceReaper.lambda$start$2(ResourceReaper.java:119)'. at org.testcontainers.utility.ResourceReaper$$Lambda$1096/0x000000010099bc40.run(Unknown Source)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "pool-7-thread-1" #139 prio=5 os_prio=0 cpu=14.51ms elapsed=50693.10s tid=0x00007f6e9400c000 nid=0x4801 waiting on condition [0x00007f6e449c0000]'. java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000c235c8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1170)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "pool-9-thread-1" #505 prio=5 os_prio=0 cpu=6.09ms elapsed=50666.17s tid=0x00007f6e6410f800 nid=0x4979 waiting on condition [0x00007f6e3df87000]'. java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000c235d748> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1170)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "ExpiringMapExpirer-1" #1393 daemon prio=5 os_prio=0 cpu=3448.31ms elapsed=50631.13s tid=0x00007f6eb0008000 nid=0x4d00 waiting on condition [ java.lang.Thread.State: TIMED_WAITING (sleeping)'. at java.lang.Thread.sleep(java.base(a)11.0.7/Native Method)'. at org.apache.mina.util.ExpiringMap$Expirer.run(ExpiringMap.java:288)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "Timer runner-3,dg-rhel7-0-58501" #2425 prio=5 os_prio=0 cpu=68592.14ms elapsed=50567.69s tid=0x00007f6ee1ff9800 nid=0x511a waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef1c58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.7/AbstractQueuedSynchronizer.java:2123)'. at java.util.concurrent.DelayQueue.take(java.base@11.0.7/DelayQueue.java:229)'. at java.util.concurrent.DelayQueue.take(java.base@11.0.7/DelayQueue.java:77)'. at org.jgroups.util.TimeScheduler3.run(TimeScheduler3.java:198)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "MPING-4,dg-rhel7-0-58501" #2426 daemon prio=5 os_prio=0 cpu=2337.48ms elapsed=50567.69s tid=0x00007f6ee279c000 nid=0x511b runnable [ java.lang.Thread.State: RUNNABLE'. at java.net.PlainDatagramSocketImpl.receive0(java.base(a)11.0.7/Native Method)'. - locked <0x00000000d0ee6520> (a java.net.PlainDatagramSocketImpl)'. at java.net.AbstractPlainDatagramSocketImpl.receive(java.base@11.0.7/AbstractPlainDatagramSocketImpl.java:181)'. - locked <0x00000000d0ee6520> (a java.net.PlainDatagramSocketImpl)'. at java.net.DatagramSocket.receive(java.base@11.0.7/DatagramSocket.java:814)'. - locked <0x00000000d0ee6550> (a java.net.DatagramPacket)'. - locked <0x00000000d0ee6578> (a java.net.MulticastSocket)'. at org.jgroups.protocols.MPING.run(MPING.java:195)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "TcpServer.Acceptor[7800]-2,dg-rhel7-0-58501" #2424 prio=5 os_prio=0 cpu=0.21ms elapsed=50567.68s tid=0x00007f6ee2697000 nid=0x511e runnable [ java.lang.Thread.State: RUNNABLE'. at java.net.PlainSocketImpl.socketAccept(java.base(a)11.0.7/Native Method)'. at java.net.AbstractPlainSocketImpl.accept(java.base@11.0.7/AbstractPlainSocketImpl.java:458)'. at java.net.ServerSocket.implAccept(java.base@11.0.7/ServerSocket.java:565)'. at java.net.ServerSocket.accept(java.base@11.0.7/ServerSocket.java:533)'. at org.jgroups.blocks.cs.TcpServer$Acceptor.run(TcpServer.java:110)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "FD_SOCK acceptor-7,dg-rhel7-0-58501" #2429 daemon prio=5 os_prio=0 cpu=0.43ms elapsed=50567.68s tid=0x00007f6ee23ec800 nid=0x511f runnable [ java.lang.Thread.State: RUNNABLE'. at java.net.PlainSocketImpl.socketAccept(java.base(a)11.0.7/Native Method)'. at java.net.AbstractPlainSocketImpl.accept(java.base@11.0.7/AbstractPlainSocketImpl.java:458)'. at java.net.ServerSocket.implAccept(java.base@11.0.7/ServerSocket.java:565)'. at java.net.ServerSocket.accept(java.base@11.0.7/ServerSocket.java:533)'. at org.jgroups.protocols.FD_SOCK$ServerSocketHandler.run(FD_SOCK.java:1014)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p77-t1" #2430 daemon prio=5 os_prio=0 cpu=89.69ms elapsed=50565.68s tid=0x00007f6ee2675800 nid=0x5120 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef2738> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p77-t2" #2431 daemon prio=5 os_prio=0 cpu=76.84ms elapsed=50565.68s tid=0x00007f6ee23f8000 nid=0x5121 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef2738> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p77-t3" #2432 daemon prio=5 os_prio=0 cpu=73.73ms elapsed=50565.68s tid=0x00007f6ee2686000 nid=0x5122 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef2738> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "timeout-thread--p79-t1" #2433 daemon prio=5 os_prio=0 cpu=236.63ms elapsed=50565.68s tid=0x00007f6eac873000 nid=0x5123 waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ee69d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.7/AbstractQueuedSynchronizer.java:2123)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1182)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p77-t4" #2434 daemon prio=5 os_prio=0 cpu=81.35ms elapsed=50565.68s tid=0x00007f6eac875800 nid=0x5124 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef2738> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "blocking-thread--p78-t1" #2435 daemon prio=5 os_prio=0 cpu=538.12ms elapsed=50565.67s tid=0x00007f6ee2674800 nid=0x5125 waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ee6dc0> (a org.jboss.threads.EnhancedQueueExecutor)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1421)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p77-t5" #2436 daemon prio=5 os_prio=0 cpu=77.19ms elapsed=50565.67s tid=0x00007f6ebc01d000 nid=0x5126 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef2738> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "expiration-thread--p80-t1" #2437 daemon prio=5 os_prio=0 cpu=3013.89ms elapsed=50565.67s tid=0x00007f6ee270a800 nid=0x5127 waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ee71a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.7/AbstractQueuedSynchronizer.java:2123)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1182)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p77-t6" #2438 daemon prio=5 os_prio=0 cpu=75.88ms elapsed=50565.67s tid=0x00007f6ee23e6800 nid=0x5128 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef2738> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p77-t7" #2439 daemon prio=5 os_prio=0 cpu=72.21ms elapsed=50565.67s tid=0x00007f6ebc003000 nid=0x5129 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef2738> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p77-t8" #2440 daemon prio=5 os_prio=0 cpu=67.05ms elapsed=50565.66s tid=0x00007f6ee27ec800 nid=0x512a waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef2738> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "HotRod-hotrod-Scheduled-49-1" #2441 prio=5 os_prio=0 cpu=492.29ms elapsed=50565.58s tid=0x00007f6e70024000 nid=0x512b waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d3594860> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.7/AbstractQueuedSynchronizer.java:2123)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1182)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "SINGLE_PORT-ServerMaster-50-1" #2442 prio=5 os_prio=0 cpu=0.49ms elapsed=50565.56s tid=0x00007f6ee2684800 nid=0x512c runnable [ java.lang.Thread.State: RUNNABLE'. at io.netty.channel.epoll.Native.epollWait(Native Method)'. at io.netty.channel.epoll.Native.epollWait(Native.java:148)'. at io.netty.channel.epoll.Native.epollWait(Native.java:141)'. at io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290)'. at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347)'. at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)'. at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)'. at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "Timer runner-3,dg-rhel7-0-9230" #2445 prio=5 os_prio=0 cpu=68572.24ms elapsed=50565.38s tid=0x00007f6ee270c000 nid=0x512f waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d4466398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.7/AbstractQueuedSynchronizer.java:2123)'. at java.util.concurrent.DelayQueue.take(java.base@11.0.7/DelayQueue.java:229)'. at java.util.concurrent.DelayQueue.take(java.base@11.0.7/DelayQueue.java:77)'. at org.jgroups.util.TimeScheduler3.run(TimeScheduler3.java:198)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "MPING-4,dg-rhel7-0-9230" #2446 daemon prio=5 os_prio=0 cpu=3138.13ms elapsed=50565.38s tid=0x00007f6ee26e1800 nid=0x5130 runnable [ java.lang.Thread.State: RUNNABLE'. at java.net.PlainDatagramSocketImpl.receive0(java.base(a)11.0.7/Native Method)'. - locked <0x00000000d44968d0> (a java.net.PlainDatagramSocketImpl)'. at java.net.AbstractPlainDatagramSocketImpl.receive(java.base@11.0.7/AbstractPlainDatagramSocketImpl.java:181)'. - locked <0x00000000d44968d0> (a java.net.PlainDatagramSocketImpl)'. at java.net.DatagramSocket.receive(java.base@11.0.7/DatagramSocket.java:814)'. - locked <0x00000000d4496900> (a java.net.DatagramPacket)'. - locked <0x00000000d4496928> (a java.net.MulticastSocket)'. at org.jgroups.protocols.MPING.run(MPING.java:195)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "TcpServer.Acceptor[7801]-2,dg-rhel7-0-9230" #2444 prio=5 os_prio=0 cpu=0.44ms elapsed=50565.37s tid=0x00007f6ee23b8000 nid=0x5133 runnable [ java.lang.Thread.State: RUNNABLE'. at java.net.PlainSocketImpl.socketAccept(java.base(a)11.0.7/Native Method)'. at java.net.AbstractPlainSocketImpl.accept(java.base@11.0.7/AbstractPlainSocketImpl.java:458)'. at java.net.ServerSocket.implAccept(java.base@11.0.7/ServerSocket.java:565)'. at java.net.ServerSocket.accept(java.base@11.0.7/ServerSocket.java:533)'. at org.jgroups.blocks.cs.TcpServer$Acceptor.run(TcpServer.java:110)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "FD_SOCK acceptor-7,dg-rhel7-0-9230" #2449 daemon prio=5 os_prio=0 cpu=0.25ms elapsed=50565.37s tid=0x00007f6ee2721800 nid=0x5134 runnable [ java.lang.Thread.State: RUNNABLE'. at java.net.PlainSocketImpl.socketAccept(java.base(a)11.0.7/Native Method)'. at java.net.AbstractPlainSocketImpl.accept(java.base@11.0.7/AbstractPlainSocketImpl.java:458)'. at java.net.ServerSocket.implAccept(java.base@11.0.7/ServerSocket.java:565)'. at java.net.ServerSocket.accept(java.base@11.0.7/ServerSocket.java:533)'. at org.jgroups.protocols.FD_SOCK$ServerSocketHandler.run(FD_SOCK.java:1014)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "Connection.Receiver [127.0.0.1:38673 - 127.0.0.1:7801]-8,dg-rhel7-0-58501" #2450 prio=5 os_prio=0 cpu=5856.24ms elapsed=50565.37s java.lang.Thread.State: RUNNABLE'. at java.net.SocketInputStream.socketRead0(java.base(a)11.0.7/Native Method)'. at java.net.SocketInputStream.socketRead(java.base@11.0.7/SocketInputStream.java:115)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:168)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:140)'. at java.io.BufferedInputStream.fill(java.base@11.0.7/BufferedInputStream.java:252)'. at java.io.BufferedInputStream.read(java.base@11.0.7/BufferedInputStream.java:271)'. - locked <0x00000000d44a8d48> (a java.io.BufferedInputStream)'. at java.io.DataInputStream.readInt(java.base@11.0.7/DataInputStream.java:392)'. at org.jgroups.blocks.cs.TcpConnection$Receiver.run(TcpConnection.java:301)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "Connection.Receiver [127.0.0.1:7801 - 127.0.0.1:38673]-8,dg-rhel7-0-9230" #2451 prio=5 os_prio=0 cpu=5244.96ms elapsed=50565.37s java.lang.Thread.State: RUNNABLE'. at java.net.SocketInputStream.socketRead0(java.base(a)11.0.7/Native Method)'. at java.net.SocketInputStream.socketRead(java.base@11.0.7/SocketInputStream.java:115)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:168)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:140)'. at java.io.BufferedInputStream.fill(java.base@11.0.7/BufferedInputStream.java:252)'. at java.io.BufferedInputStream.read(java.base@11.0.7/BufferedInputStream.java:271)'. - locked <0x00000000d4468ee8> (a java.io.BufferedInputStream)'. at java.io.DataInputStream.readInt(java.base@11.0.7/DataInputStream.java:392)'. at org.jgroups.blocks.cs.TcpConnection$Receiver.run(TcpConnection.java:301)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "FD_SOCK pinger-9,dg-rhel7-0-58501" #2452 daemon prio=5 os_prio=0 cpu=1.11ms elapsed=50565.36s tid=0x00007f6ea00aa800 nid=0x5137 runnable [ java.lang.Thread.State: RUNNABLE'. at java.net.SocketInputStream.socketRead0(java.base(a)11.0.7/Native Method)'. at java.net.SocketInputStream.socketRead(java.base@11.0.7/SocketInputStream.java:115)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:168)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:140)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:200)'. at org.jgroups.protocols.FD_SOCK.run(FD_SOCK.java:430)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "notification-thread--p76-t1" #2453 daemon prio=5 os_prio=0 cpu=1.69ms elapsed=50565.36s tid=0x00007f6ea00b8000 nid=0x5138 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d44a8f28> (a org.jboss.threads.EnhancedQueueExecutor)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1409)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "FD_SOCK conn-handler-9,dg-rhel7-0-9230" #2454 daemon prio=5 os_prio=0 cpu=0.22ms elapsed=50565.36s tid=0x00007f6e70026000 nid=0x5139 runnable [ java.lang.Thread.State: RUNNABLE'. at java.net.SocketInputStream.socketRead0(java.base(a)11.0.7/Native Method)'. at java.net.SocketInputStream.socketRead(java.base@11.0.7/SocketInputStream.java:115)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:168)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:140)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:200)'. at org.jgroups.protocols.FD_SOCK$ClientConnectionHandler.run(FD_SOCK.java:1076)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "FD_SOCK pinger-10,dg-rhel7-0-9230" #2455 daemon prio=5 os_prio=0 cpu=0.85ms elapsed=50565.36s tid=0x00007f6ee2720000 nid=0x513a runnable [ java.lang.Thread.State: RUNNABLE'. at java.net.SocketInputStream.socketRead0(java.base(a)11.0.7/Native Method)'. at java.net.SocketInputStream.socketRead(java.base@11.0.7/SocketInputStream.java:115)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:168)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:140)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:200)'. at org.jgroups.protocols.FD_SOCK.run(FD_SOCK.java:430)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p82-t1" #2456 daemon prio=5 os_prio=0 cpu=80.40ms elapsed=50565.36s tid=0x00007f6ee23ed800 nid=0x513b waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d44692d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p82-t2" #2457 daemon prio=5 os_prio=0 cpu=82.88ms elapsed=50565.36s tid=0x00007f6ee27b5800 nid=0x513c waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d44692d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p82-t3" #2458 daemon prio=5 os_prio=0 cpu=74.72ms elapsed=50565.36s tid=0x00007f6ee27b6000 nid=0x513d waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d44692d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "timeout-thread--p84-t1" #2461 daemon prio=5 os_prio=0 cpu=254.86ms elapsed=50565.36s tid=0x00007f6ee278c000 nid=0x513f waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d44a9438> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.7/AbstractQueuedSynchronizer.java:2123)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1182)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "FD_SOCK conn-handler-12,dg-rhel7-0-58501" #2462 daemon prio=5 os_prio=0 cpu=0.16ms elapsed=50565.35s tid=0x00007f6ea8ba8800 nid=0x5141 runnable [ java.lang.Thread.State: RUNNABLE'. at java.net.SocketInputStream.socketRead0(java.base(a)11.0.7/Native Method)'. at java.net.SocketInputStream.socketRead(java.base@11.0.7/SocketInputStream.java:115)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:168)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:140)'. at java.net.SocketInputStream.read(java.base@11.0.7/SocketInputStream.java:200)'. at org.jgroups.protocols.FD_SOCK$ClientConnectionHandler.run(FD_SOCK.java:1076)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "blocking-thread--p83-t1" #2463 daemon prio=5 os_prio=0 cpu=524.46ms elapsed=50565.34s tid=0x00007f6ee278c800 nid=0x5142 waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d495a578> (a org.jboss.threads.EnhancedQueueExecutor)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1421)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p82-t4" #2464 daemon prio=5 os_prio=0 cpu=76.00ms elapsed=50565.34s tid=0x00007f6eac876800 nid=0x5143 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d44692d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "expiration-thread--p85-t1" #2465 daemon prio=5 os_prio=0 cpu=2736.79ms elapsed=50565.34s tid=0x00007f6ee2707000 nid=0x5144 waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d495f638> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.7/AbstractQueuedSynchronizer.java:2123)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1182)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p82-t5" #2466 daemon prio=5 os_prio=0 cpu=80.34ms elapsed=50565.34s tid=0x00007f6ee2708000 nid=0x5145 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d44692d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p82-t6" #2467 daemon prio=5 os_prio=0 cpu=69.20ms elapsed=50565.34s tid=0x00007f6eac877800 nid=0x5146 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d44692d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p82-t7" #2468 daemon prio=5 os_prio=0 cpu=72.16ms elapsed=50565.33s tid=0x00007f6ee2708800 nid=0x5147 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d44692d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "non-blocking-thread--p82-t8" #2469 daemon prio=5 os_prio=0 cpu=79.82ms elapsed=50565.33s tid=0x00007f6e684fb800 nid=0x5148 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d44692d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.7/AbstractQueuedSynchronizer.java:2081)'. at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.7/LinkedBlockingQueue.java:433)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "blocking-thread--p78-t2" #2470 daemon prio=5 os_prio=0 cpu=3.50ms elapsed=50565.30s tid=0x00007f6e9c02e000 nid=0x5149 waiting on condition [ java.lang.Thread.State: WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ee6dc0> (a org.jboss.threads.EnhancedQueueExecutor)'. at java.util.concurrent.locks.LockSupport.park(java.base@11.0.7/LockSupport.java:194)'. at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1409)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "HotRod-hotrod-Scheduled-52-1" #2472 prio=5 os_prio=0 cpu=537.77ms elapsed=50565.16s tid=0x00007f6e7000c000 nid=0x514b waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0f8ef30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.7/AbstractQueuedSynchronizer.java:2123)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:1182)'. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.7/ScheduledThreadPoolExecutor.java:899)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1054)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "SINGLE_PORT-ServerMaster-53-1" #2473 prio=5 os_prio=0 cpu=0.44ms elapsed=50565.13s tid=0x00007f6ee27e0000 nid=0x514c runnable [ java.lang.Thread.State: RUNNABLE'. at io.netty.channel.epoll.Native.epollWait(Native Method)'. at io.netty.channel.epoll.Native.epollWait(Native.java:148)'. at io.netty.channel.epoll.Native.epollWait(Native.java:141)'. at io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:290)'. at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:347)'. at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)'. at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)'. at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "jgroups-279,dg-rhel7-0-58501" #3000 prio=5 os_prio=0 cpu=399.90ms elapsed=621.98s tid=0x00007f6e8cb8c000 nid=0x1dfd waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef1e38> (a java.util.concurrent.SynchronousQueue$TransferStack)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@11.0.7/SynchronousQueue.java:462)'. at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@11.0.7/SynchronousQueue.java:361)'. at java.util.concurrent.SynchronousQueue.poll(java.base@11.0.7/SynchronousQueue.java:937)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1053)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "jgroups-280,dg-rhel7-0-58501" #3001 prio=5 os_prio=0 cpu=275.75ms elapsed=449.71s tid=0x00007f6e7c223800 nid=0x1e03 waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d0ef1e38> (a java.util.concurrent.SynchronousQueue$TransferStack)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@11.0.7/SynchronousQueue.java:462)'. at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@11.0.7/SynchronousQueue.java:361)'. at java.util.concurrent.SynchronousQueue.poll(java.base@11.0.7/SynchronousQueue.java:937)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1053)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "jgroups-270,dg-rhel7-0-9230" #3002 prio=5 os_prio=0 cpu=258.25ms elapsed=438.17s tid=0x00007f6e641e6000 nid=0x1e04 waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d4466578> (a java.util.concurrent.SynchronousQueue$TransferStack)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@11.0.7/SynchronousQueue.java:462)'. at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@11.0.7/SynchronousQueue.java:361)'. at java.util.concurrent.SynchronousQueue.poll(java.base@11.0.7/SynchronousQueue.java:937)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1053)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "jgroups-272,dg-rhel7-0-9230" #3004 prio=5 os_prio=0 cpu=59.60ms elapsed=104.59s tid=0x00007f6e641e0000 nid=0x1e3c waiting on condition [ java.lang.Thread.State: TIMED_WAITING (parking)'. at jdk.internal.misc.Unsafe.park(java.base(a)11.0.7/Native Method)'. - parking to wait for <0x00000000d4466578> (a java.util.concurrent.SynchronousQueue$TransferStack)'. at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.7/LockSupport.java:234)'. at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(java.base@11.0.7/SynchronousQueue.java:462)'. at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@11.0.7/SynchronousQueue.java:361)'. at java.util.concurrent.SynchronousQueue.poll(java.base@11.0.7/SynchronousQueue.java:937)'. at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.7/ThreadPoolExecutor.java:1053)'. at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.7/ThreadPoolExecutor.java:1114)'. at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.7/ThreadPoolExecutor.java:628)'. at java.lang.Thread.run(java.base@11.0.7/Thread.java:834)'. "VM Thread" os_prio=0 cpu=4460.54ms elapsed=50777.62s tid=0x00007f6ee018b800 nid=0x4673 runnable '. "GC Thread#0" os_prio=0 cpu=4377.99ms elapsed=50777.64s tid=0x00007f6ee0030000 nid=0x466e runnable '. "GC Thread#1" os_prio=0 cpu=4261.35ms elapsed=50777.47s tid=0x00007f6eb8001000 nid=0x467c runnable '. "GC Thread#2" os_prio=0 cpu=4121.76ms elapsed=50766.12s tid=0x00007f6eb8010000 nid=0x4690 runnable '. "GC Thread#3" os_prio=0 cpu=4134.90ms elapsed=50766.12s tid=0x00007f6eb8011800 nid=0x4691 runnable '. "G1 Main Marker" os_prio=0 cpu=114.00ms elapsed=50777.64s tid=0x00007f6ee004c000 nid=0x466f runnable '. "G1 Conc#0" os_prio=0 cpu=82894.96ms elapsed=50777.64s tid=0x00007f6ee004e000 nid=0x4670 runnable '. "G1 Refine#0" os_prio=0 cpu=4.70ms elapsed=50777.64s tid=0x00007f6ee00c3000 nid=0x4671 runnable '. "G1 Young RemSet Sampling" os_prio=0 cpu=12682.96ms elapsed=50777.64s tid=0x00007f6ee00c4800 nid=0x4672 runnable '. "VM Periodic Task Thread" os_prio=0 cpu=53568.96ms elapsed=50777.50s tid=0x00007f6ee02a6800 nid=0x467b waiting on condition '. JNI global refs: 43, weak refs: 45'. Heap'. garbage-first heap total 363520K, used 346188K [0x00000000c0000000, 0x0000000100000000)'. region size 1024K, 15 young (15360K), 1 survivors (1024K)'. Metaspace used 101775K, capacity 104013K, committed 106192K, reserved 1142784K'. class space used 12408K, capacity 13319K, committed 13824K, reserved 1048576K'. {noformat}