Leaked thread: mysql-cj-abandoned-connection-cleanup ---------------------------------------------------- Key: ISPN-12321 URL: https://issues.redhat.com/browse/ISPN-12321 Project: Infinispan Issue Type: Bug Components: Loaders and Stores, Test Suite Affects Versions: 12.0.0.Dev03 Reporter: Dan Berindei Assignee: Dan Berindei Priority: Major Fix For: 12.0.0.Dev04 After the mysql 8.0.16 upgrade suggested by dependabot, we got a new thread leak. The newer mysql driver starts a thread that checks for connection leaks, and the thread keeps running when the test suite has finished: {noformat} org.infinispan.commons.test.ThreadLeakChecker$LeakException: Leaked thread: mysql-cj-abandoned-connection-cleanup << blocking-thread-Test-NodeA-p3-t1 << testng-Test << org.infinispan.persistence.jdbc.stringbased.StringStoreWithManagedConnectionFunctionalTest at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) ~[?:?] at com.mysql.cj.jdbc.AbandonedConnectionCleanupThread.run(AbandonedConnectionCleanupThread.java:85) ~[mysql-connector-java-8.0.16.jar:8.0.16] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) ~[?:?] at java.lang.Thread.run(Thread.java:832) ~[?:?] Caused by: org.infinispan.commons.test.ThreadLeakChecker$LeakException: blocking-thread-Test-NodeA-p3-t1 << testng-Test << org.infinispan.persistence.jdbc.stringbased.StringStoreWithManagedConnectionFunctionalTest at org.infinispan.commons.test.ThreadLeakChecker$ThreadInfoLocal.childValue(ThreadLeakChecker.java:109) ~[classes/:?] at org.infinispan.commons.test.ThreadLeakChecker$ThreadInfoLocal.childValue(ThreadLeakChecker.java:106) ~[classes/:?] at java.lang.ThreadLocal$ThreadLocalMap.<init>(ThreadLocal.java:411) ~[?:?] at java.lang.ThreadLocal.createInheritedMap(ThreadLocal.java:276) ~[?:?] at java.lang.Thread.<init>(Thread.java:448) ~[?:?] at java.lang.Thread.<init>(Thread.java:707) ~[?:?] at java.lang.Thread.<init>(Thread.java:580) ~[?:?] at com.mysql.cj.jdbc.AbandonedConnectionCleanupThread.lambda$static$0(AbandonedConnectionCleanupThread.java:59) ~[mysql-connector-java-8.0.16.jar:8.0.16] at java.util.concurrent.ThreadPoolExecutor$Worker.<init>(ThreadPoolExecutor.java:625) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:914) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1345) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:687) ~[?:?] at com.mysql.cj.jdbc.AbandonedConnectionCleanupThread.<clinit>(AbandonedConnectionCleanupThread.java:75) ~[mysql-connector-java-8.0.16.jar:8.0.16] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:340) ~[?:?] at com.mysql.cj.jdbc.NonRegisteringDriver.<clinit>(NonRegisteringDriver.java:99) ~[mysql-connector-java-8.0.16.jar:8.0.16] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:500) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:481) ~[?:?] at java.util.ServiceLoader$ProviderImpl.newInstance(ServiceLoader.java:782) ~[?:?] at java.util.ServiceLoader$ProviderImpl.get(ServiceLoader.java:724) ~[?:?] at java.util.ServiceLoader$3.next(ServiceLoader.java:1396) ~[?:?] at java.sql.DriverManager$2.run(DriverManager.java:614) ~[java.sql:?] at java.sql.DriverManager$2.run(DriverManager.java:594) ~[java.sql:?] at java.security.AccessController.doPrivileged(AccessController.java:312) ~[?:?] at java.sql.DriverManager.ensureDriversInitialized(DriverManager.java:594) ~[java.sql:?] at java.sql.DriverManager.getConnection(DriverManager.java:665) ~[java.sql:?] at java.sql.DriverManager.getConnection(DriverManager.java:228) ~[java.sql:?] at org.infinispan.persistence.jdbc.impl.connectionfactory.SimpleConnectionFactory.getConnection(SimpleConnectionFactory.java:62) ~[classes/:?] at org.infinispan.persistence.jdbc.impl.table.TableManagerFactory.getDbMetaData(TableManagerFactory.java:71) ~[classes/:?] at org.infinispan.persistence.jdbc.impl.table.TableManagerFactory.getManager(TableManagerFactory.java:28) ~[classes/:?] at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.getTableManager(JdbcStringBasedStore.java:789) ~[classes/:?] at org.infinispan.persistence.jdbc.stringbased.JdbcStringBasedStore.start(JdbcStringBasedStore.java:128) ~[classes/:?] at org.infinispan.persistence.support.NonBlockingStoreAdapter.lambda$start$0(NonBlockingStoreAdapter.java:108) ~[classes/:?] at org.infinispan.util.concurrent.BlockingManagerImpl.lambda$runBlockingOperation$2(BlockingManagerImpl.java:121) ~[classes/:?] at java.util.concurrent.CompletableFuture$AsyncRun.run$$$capture(CompletableFuture.java:1800) ~[?:?] at java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java) ~[?:?] at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35) ~[jboss-threads-2.3.3.Final.jar:2.3.3.Final] at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1982) ~[jboss-threads-2.3.3.Final.jar:2.3.3.Final] at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1486) ~[jboss-threads-2.3.3.Final.jar:2.3.3.Final] at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1348) ~[jboss-threads-2.3.3.Final.jar:2.3.3.Final] ... 1 more Caused by: org.infinispan.commons.test.ThreadLeakChecker$LeakException: testng-Test << org.infinispan.persistence.jdbc.stringbased.StringStoreWithManagedConnectionFunctionalTest at org.infinispan.commons.test.ThreadLeakChecker$ThreadInfoLocal.childValue(ThreadLeakChecker.java:109) ~[classes/:?] at org.infinispan.commons.test.ThreadLeakChecker$ThreadInfoLocal.childValue(ThreadLeakChecker.java:106) ~[classes/:?] at java.lang.ThreadLocal$ThreadLocalMap.<init>(ThreadLocal.java:411) ~[?:?] at java.lang.ThreadLocal.createInheritedMap(ThreadLocal.java:276) ~[?:?] at java.lang.Thread.<init>(Thread.java:448) ~[?:?] at java.lang.Thread.<init>(Thread.java:707) ~[?:?] at java.lang.Thread.<init>(Thread.java:528) ~[?:?] at org.infinispan.commons.jdkspecific.ThreadCreator.createThread(ThreadCreator.java:9) ~[classes/:?] at org.infinispan.factories.threads.DefaultThreadFactory.actualThreadCreate(DefaultThreadFactory.java:113) ~[classes/:?] at org.infinispan.factories.threads.DefaultThreadFactory.createThread(DefaultThreadFactory.java:105) ~[classes/:?] at org.infinispan.factories.threads.DefaultThreadFactory.newThread(DefaultThreadFactory.java:99) ~[classes/:?] at org.jboss.threads.EnhancedQueueExecutor.doStartThread(EnhancedQueueExecutor.java:1599) ~[jboss-threads-2.3.3.Final.jar:2.3.3.Final] at org.jboss.threads.EnhancedQueueExecutor.execute(EnhancedQueueExecutor.java:751) ~[jboss-threads-2.3.3.Final.jar:2.3.3.Final] at org.infinispan.util.concurrent.BlockingTaskAwareExecutorServiceImpl.execute(BlockingTaskAwareExecutorServiceImpl.java:116) ~[classes/:?] at org.infinispan.executors.LazyInitializingBlockingTaskAwareExecutorService.execute(LazyInitializingBlockingTaskAwareExecutorService.java:194) ~[classes/:?] at java.util.concurrent.CompletableFuture.asyncRunStage(CompletableFuture.java:1814) ~[?:?] at java.util.concurrent.CompletableFuture.runAsync(CompletableFuture.java:2023) ~[?:?] at org.infinispan.util.concurrent.BlockingManagerImpl.runBlockingOperation(BlockingManagerImpl.java:119) ~[classes/:?] at org.infinispan.util.concurrent.BlockingManagerImpl.runBlockingOperation(BlockingManagerImpl.java:100) ~[classes/:?] at org.infinispan.util.concurrent.BlockingManagerImpl.runBlocking(BlockingManagerImpl.java:66) ~[classes/:?] at org.infinispan.persistence.support.NonBlockingStoreAdapter.start(NonBlockingStoreAdapter.java:102) ~[classes/:?] at org.infinispan.persistence.manager.PersistenceManagerImpl.lambda$start$2(PersistenceManagerImpl.java:203) ~[classes/:?] at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapSingle$ConcatMapSingleSubscriber.drain(FlowableConcatMapSingle.java:249) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapSingle$ConcatMapSingleSubscriber.onNext(FlowableConcatMapSingle.java:136) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable$IteratorSubscription.slowPath(FlowableFromIterable.java:243) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable$BaseRangeSubscription.request(FlowableFromIterable.java:131) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapSingle$ConcatMapSingleSubscriber.onSubscribe(FlowableConcatMapSingle.java:125) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable.subscribe(FlowableFromIterable.java:69) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.flowable.FlowableFromIterable.subscribeActual(FlowableFromIterable.java:47) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.mixed.FlowableConcatMapSingle.subscribeActual(FlowableConcatMapSingle.java:61) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.flowable.FlowableDoOnEach.subscribeActual(FlowableDoOnEach.java:50) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.flowable.FlowableFlatMap.subscribeActual(FlowableFlatMap.java:52) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.core.Flowable.subscribe(Flowable.java:15750) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.flowable.FlowableIgnoreElementsCompletable.subscribeActual(FlowableIgnoreElementsCompletable.java:34) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.core.Completable.subscribe(Completable.java:2850) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.internal.operators.completable.CompletablePeek.subscribeActual(CompletablePeek.java:51) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.core.Completable.subscribe(Completable.java:2850) ~[rxjava-3.0.4.jar:?] at io.reactivex.rxjava3.core.Completable.blockingAwait(Completable.java:1460) ~[rxjava-3.0.4.jar:?] at org.infinispan.persistence.manager.PersistenceManagerImpl.start(PersistenceManagerImpl.java:230) ~[classes/:?] at org.infinispan.persistence.support.DelegatingPersistenceManager.start(DelegatingPersistenceManager.java:43) ~[classes/:?] at org.infinispan.persistence.support.CorePackageImpl$1.start(CorePackageImpl.java:33) ~[classes/:?] at org.infinispan.persistence.support.CorePackageImpl$1.start(CorePackageImpl.java:27) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:592) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:589) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:583) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:552) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:775) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:610) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:574) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:552) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:775) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:610) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:574) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:552) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:775) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:610) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:574) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:552) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:775) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:610) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:574) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:552) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:775) ~[classes/:?] at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:341) ~[classes/:?] at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:237) ~[classes/:?] at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:213) ~[classes/:?] at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1007) ~[classes/:?] at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:512) ~[classes/:?] at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:697) ~[classes/:?] at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:643) ~[classes/:?] at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:532) ~[classes/:?] at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:510) ~[classes/:?] at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26) ~[classes/:?] at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14) ~[classes/:?] at org.infinispan.security.Security.doPrivileged(Security.java:46) ~[classes/:?] at org.infinispan.globalstate.impl.SecurityActions.doPrivileged(SecurityActions.java:29) ~[classes/:?] at org.infinispan.globalstate.impl.SecurityActions.getCache(SecurityActions.java:38) ~[classes/:?] at org.infinispan.globalstate.impl.GlobalConfigurationManagerImpl.start(GlobalConfigurationManagerImpl.java:89) ~[classes/:?] at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:59) ~[classes/:?] at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:48) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:592) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:583) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:552) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30) ~[classes/:?] at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:775) ~[classes/:?] at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:341) ~[classes/:?] at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:237) ~[classes/:?] at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:746) ~[classes/:?] at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:717) ~[classes/:?] at org.infinispan.manager.DefaultCacheManager.<init>(DefaultCacheManager.java:398) ~[classes/:?] at org.infinispan.test.fwk.TestCacheManagerFactory.newDefaultCacheManager(TestCacheManagerFactory.java:83) ~[test-classes/:?] at org.infinispan.persistence.jdbc.stringbased.AbstractStringBasedCacheStore.configureCacheManager(AbstractStringBasedCacheStore.java:135) ~[test-classes/:?] at org.infinispan.persistence.jdbc.stringbased.AbstractStringBasedCacheStore.testPutGetRemoveWithPassivationWithoutPreload(AbstractStringBasedCacheStore.java:81) ~[test-classes/:?] {noformat}