Remoting deadlock in org.jboss.as.testsuite.integration.secman.subsystem.MinimumPermissionsTestCase --------------------------------------------------------------------------------------------------- Key: WFCORE-1660 URL: https://issues.jboss.org/browse/WFCORE-1660 Project: WildFly Core Issue Type: Bug Affects Versions: 2.2.0.CR8 Reporter: Brian Stansberry Assignee: David Lloyd Priority: Critical Hang in the WildFly full testsuite running a core 2.x-ignore test. No commits in 2.x-ignore that seem relevant. Both WildFly Core 2.x and master are using Remoting 4.0.21. Looking at the deadlock this appears to be a problem in remoting. https://ci.wildfly.org/viewLog.html?buildTypeId=WF_WildFlyCore2xMasterIgn... {code} Found one Java-level deadlock: ============================= "Remoting "cli-client" task-6": waiting to lock monitor 0xbef755e0 (object 0xe60ca518, a java.lang.Object), which is held by "main" "main": waiting to lock monitor 0xc6d1ea50 (object 0xe0fbcab8, a java.lang.Object), which is held by "Remoting "cli-client" task-6" Java stack information for the threads listed above: =================================================== "Remoting "cli-client" task-6": at org.jboss.remoting3.EndpointImpl$4.setResult(EndpointImpl.java:323) - waiting to lock <0xe60ca518> (a java.lang.Object) at org.jboss.remoting3.EndpointImpl$4.setResult(EndpointImpl.java:295) at org.xnio.IoUtils$ResultNotifier.handleDone(IoUtils.java:706) at org.xnio.IoUtils$ResultNotifier.handleDone(IoUtils.java:695) at org.xnio.IoFuture$HandlingNotifier.notify(IoFuture.java:208) at org.xnio.AbstractIoFuture$1.run(AbstractIoFuture.java:211) at org.xnio.IoUtils$2.execute(IoUtils.java:70) at org.xnio.AbstractIoFuture.runNotifier(AbstractIoFuture.java:354) at org.xnio.AbstractIoFuture.runAllNotifiers(AbstractIoFuture.java:233) at org.xnio.AbstractIoFuture.setResult(AbstractIoFuture.java:272) - locked <0xe0fbcab8> (a java.lang.Object) at org.xnio.FutureResult.setResult(FutureResult.java:79) at org.jboss.remoting3.remote.ClientConnectionOpenListener$Authentication$2.run(ClientConnectionOpenListener.java:755) at org.jboss.remoting3.EndpointImpl$TrackingExecutor$1.run(EndpointImpl.java:731) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "main": at org.xnio.AbstractIoFuture.addCancelHandler(AbstractIoFuture.java:329) - waiting to lock <0xe0fbcab8> (a java.lang.Object) at org.xnio.FutureResult.addCancelHandler(FutureResult.java:69) at org.jboss.remoting3.remote.RemoteConnectionProvider.connect(RemoteConnectionProvider.java:211) at org.jboss.remoting3.EndpointImpl.doConnect(EndpointImpl.java:340) - locked <0xe60ca518> (a java.lang.Object) at org.jboss.remoting3.EndpointImpl.doConnect(EndpointImpl.java:276) at org.jboss.remoting3.EndpointImpl.connect(EndpointImpl.java:393) at org.jboss.remoting3.EndpointImpl.connect(EndpointImpl.java:381) at org.jboss.as.protocol.ProtocolConnectionUtils.connect(ProtocolConnectionUtils.java:83) at org.jboss.as.protocol.ProtocolConnectionUtils.connectSync(ProtocolConnectionUtils.java:114) at org.jboss.as.protocol.ProtocolConnectionManager$EstablishingConnection.connect(ProtocolConnectionManager.java:257) at org.jboss.as.protocol.ProtocolConnectionManager.connect(ProtocolConnectionManager.java:71) - locked <0xe0fba058> (a org.jboss.as.protocol.ProtocolConnectionManager) at org.jboss.as.protocol.mgmt.FutureManagementChannel$Establishing.getChannel(FutureManagementChannel.java:218) at org.jboss.as.cli.impl.CLIModelControllerClient.getOrCreateChannel(CLIModelControllerClient.java:179) - locked <0xddab9fe0> (a java.lang.Object) at org.jboss.as.cli.impl.CLIModelControllerClient$3.getChannel(CLIModelControllerClient.java:138) at org.jboss.as.protocol.mgmt.ManagementChannelHandler.executeRequest(ManagementChannelHandler.java:147) at org.jboss.as.protocol.mgmt.ManagementChannelHandler.executeRequest(ManagementChannelHandler.java:122) at org.jboss.as.controller.client.impl.AbstractModelControllerClient.executeRequest(AbstractModelControllerClient.java:263) at org.jboss.as.controller.client.impl.AbstractModelControllerClient.execute(AbstractModelControllerClient.java:168) at org.jboss.as.controller.client.impl.AbstractModelControllerClient.executeForResult(AbstractModelControllerClient.java:147) at org.jboss.as.controller.client.impl.AbstractModelControllerClient.execute(AbstractModelControllerClient.java:75) at org.jboss.as.cli.Util.validateRequest(Util.java:1147) at org.jboss.as.cli.handlers.OperationRequestHandler.handle(OperationRequestHandler.java:72) at org.jboss.as.cli.impl.CommandContextImpl.handle(CommandContextImpl.java:753) at org.jboss.as.test.integration.management.util.CLIWrapper.sendLine(CLIWrapper.java:163) at org.jboss.as.test.integration.management.util.CLIWrapper.sendLine(CLIWrapper.java:187) at org.jboss.as.testsuite.integration.secman.subsystem.ReloadableCliTestBase.doCliOperation(ReloadableCliTestBase.java:46) at org.jboss.as.testsuite.integration.secman.subsystem.MinimumPermissionsTestCase.testFilePerm(MinimumPermissionsTestCase.java:89) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.jboss.arquillian.junit.Arquillian$8$1.invoke(Arquillian.java:370) at org.jboss.arquillian.container.test.impl.execution.LocalTestExecuter.execute(LocalTestExecuter.java:60) at sun.reflect.GeneratedMethodAccessor14.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.invokeObservers(EventContextImpl.java:99) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:81) at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:145) at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:116) at org.jboss.arquillian.core.impl.EventImpl.fire(EventImpl.java:67) at org.jboss.arquillian.container.test.impl.execution.ClientTestExecuter.execute(ClientTestExecuter.java:53) at sun.reflect.GeneratedMethodAccessor12.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.invokeObservers(EventContextImpl.java:99) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:81) at org.jboss.arquillian.container.test.impl.client.ContainerEventController.createContext(ContainerEventController.java:142) at org.jboss.arquillian.container.test.impl.client.ContainerEventController.createTestContext(ContainerEventController.java:129) at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:88) at org.jboss.arquillian.test.impl.TestContextHandler.createClassContext(TestContextHandler.java:92) at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:88) at org.jboss.arquillian.test.impl.TestContextHandler.createTestContext(TestContextHandler.java:130) at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:88) at org.jboss.arquillian.test.impl.TestContextHandler.createSuiteContext(TestContextHandler.java:73) at sun.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:88) at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:145) at org.jboss.arquillian.test.impl.EventTestRunnerAdaptor.test(EventTestRunnerAdaptor.java:136) at org.jboss.arquillian.junit.Arquillian$8.evaluate(Arquillian.java:363) at org.jboss.arquillian.junit.Arquillian$4.evaluate(Arquillian.java:245) at org.jboss.arquillian.junit.Arquillian.multiExecute(Arquillian.java:422) at org.jboss.arquillian.junit.Arquillian.access$200(Arquillian.java:54) at org.jboss.arquillian.junit.Arquillian$5.evaluate(Arquillian.java:259) at org.jboss.arquillian.junit.Arquillian$7$1.invoke(Arquillian.java:315) at org.jboss.arquillian.container.test.impl.execution.ClientBeforeAfterLifecycleEventExecuter.execute(ClientBeforeAfterLifecycleEventExecuter.java:99) at org.jboss.arquillian.container.test.impl.execution.ClientBeforeAfterLifecycleEventExecuter.on(ClientBeforeAfterLifecycleEventExecuter.java:72) at sun.reflect.GeneratedMethodAccessor5.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.invokeObservers(EventContextImpl.java:99) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:81) at org.jboss.arquillian.container.test.impl.client.ContainerEventController.createContext(ContainerEventController.java:142) at org.jboss.arquillian.container.test.impl.client.ContainerEventController.createBeforeContext(ContainerEventController.java:124) at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:88) at org.jboss.arquillian.test.impl.TestContextHandler.createClassContext(TestContextHandler.java:92) at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:88) at org.jboss.arquillian.test.impl.TestContextHandler.createTestContext(TestContextHandler.java:130) at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:88) at org.jboss.arquillian.test.impl.TestContextHandler.createSuiteContext(TestContextHandler.java:73) at sun.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.arquillian.core.impl.ObserverImpl.invoke(ObserverImpl.java:94) at org.jboss.arquillian.core.impl.EventContextImpl.proceed(EventContextImpl.java:88) at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:145) at org.jboss.arquillian.core.impl.ManagerImpl.fire(ManagerImpl.java:116) at org.jboss.arquillian.test.impl.EventTestRunnerAdaptor.fireCustomLifecycle(EventTestRunnerAdaptor.java:159) at org.jboss.arquillian.junit.Arquillian$7.evaluate(Arquillian.java:311) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.jboss.arquillian.junit.Arquillian$2.evaluate(Arquillian.java:204) at org.jboss.arquillian.junit.Arquillian.multiExecute(Arquillian.java:422) at org.jboss.arquillian.junit.Arquillian.access$200(Arquillian.java:54) at org.jboss.arquillian.junit.Arquillian$3.evaluate(Arquillian.java:218) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.jboss.arquillian.junit.Arquillian.run(Arquillian.java:166) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) at org.junit.runners.ParentRunner.run(ParentRunner.java:363) at org.junit.runner.JUnitCore.run(JUnitCore.java:137) at org.junit.runner.JUnitCore.run(JUnitCore.java:115) at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.createRequestAndRun(JUnitCoreWrapper.java:107) at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.executeEager(JUnitCoreWrapper.java:77) at org.apache.maven.surefire.junitcore.JUnitCoreWrapper.execute(JUnitCoreWrapper.java:53) at org.apache.maven.surefire.junitcore.JUnitCoreProvider.invoke(JUnitCoreProvider.java:144) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:203) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:155) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103) Found 1 deadlock. {code}