deadlock during cluster failover -------------------------------- Key: JBAS-1151 URL: http://jira.jboss.com/jira/browse/JBAS-1151 Project: JBoss Application Server Issue Type: Bug Security Level: Public(Everyone can see) Components: Clustering Affects Versions: JBossAS-3.2.6 Final Reporter: SourceForge User Assigned To: Brian Stansberry Fix For: JBossAS-5.0.0.CR1 SourceForge Submitter: eiman . We are running JBoss 3.2.5 on jdk 1.4.2 running on redhat linux. In our environment we are running a two node cluster (node A and node B). We restarted node A and then within a minute restarted node B. Upon doing that we found a deadlock in node B: Java stack information for the threads listed above: =================================================== "MessageDispatcher up processing thread": at org.jboss.system.ServiceController.create(ServiceController.java:229) - waiting to lock <0x56505440> (a org.jboss.system.ServiceController) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:61) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:53) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473) at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176) at $Proxy33.create(Unknown Source) at org.jboss.web.AbstractWebContainer.create(AbstractWebContainer.java:291) at org.jboss.deployment.MainDeployer.create(MainDeployer.java:790) at org.jboss.deployment.MainDeployer.create(MainDeployer.java:782) at org.jboss.deployment.MainDeployer.create(MainDeployer.java:782) at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:644) at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:608) at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:592) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:61) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:53) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473) at org.jboss.ha.singleton.HASingletonController.invokeSingletonMBeanMethod(HASingletonController.java:114) at org.jboss.ha.singleton.HASingletonController.startSingleton(HASingletonController.java:63) at org.jboss.ha.singleton.HASingletonSupport.makeThisNodeMaster(HASingletonSupport.java:137) at org.jboss.ha.singleton.HASingletonSupport.partitionTopologyChanged(HASingletonSupport.java:118) at org.jboss.ha.jmx.HAServiceMBeanSupport$1.replicantsChanged(HAServiceMBeanSupport.java:197) at org.jboss.ha.framework.server.DistributedReplicantManagerImpl.notifyKeyListeners(DistributedReplicantManagerImpl.java:701) - locked <0x56609350> (a java.util.HashMap) at org.jboss.ha.framework.server.DistributedReplicantManagerImpl._remove(DistributedReplicantManagerImpl.java:512) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jgroups.blocks.MethodCall.invoke(MethodCall.java:236) at org.jboss.ha.framework.server.HAPartitionImpl.handle(HAPartitionImpl.java:826) at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:615) at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:512) at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:326) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.handleUp(MessageDispatcher.java:722) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.access$300(MessageDispatcher.java:554) at org.jgroups.blocks.MessageDispatcher$1.run(MessageDispatcher.java:691) at java.lang.Thread.run(Thread.java:534) "main": at org.jboss.ha.framework.server.DistributedReplicantManagerImpl.registerListener(DistributedReplicantManagerImpl.java:398) - waiting to lock <0x56609350> (a java.util.HashMap) at org.jboss.ha.framework.server.HATarget.updateHAPartition(HATarget.java:130) at org.jboss.ha.framework.server.HATarget.<init>(HATarget.java:58) at org.jboss.proxy.ejb.ProxyFactoryHA.setupInvokers(ProxyFactoryHA.java:102) at org.jboss.proxy.ejb.ProxyFactory.start(ProxyFactory.java:186) at org.jboss.proxy.ejb.ProxyFactoryHA.start(ProxyFactoryHA.java:85) at org.jboss.ejb.StatelessSessionContainer.startService(StatelessSessionContainer.java:205) at org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:271) at org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:221) at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:61) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:53) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473) at org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:837) at $Proxy17.start(Unknown Source) at org.jboss.system.ServiceController.start(ServiceController.java:367) - locked <0x56505440> (a org.jboss.system.ServiceController) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:61) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:53) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473) at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176) at $Proxy48.start(Unknown Source) at org.jboss.ejb.EjbModule.startService(EjbModule.java:367) at org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:271) at org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:221) at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:61) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:53) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473) at org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:837) at $Proxy17.start(Unknown Source) at org.jboss.system.ServiceController.start(ServiceController.java:367) - locked <0x56505440> (a org.jboss.system.ServiceController) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:61) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:53) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473) at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176) at $Proxy14.start(Unknown Source) at org.jboss.ejb.EJBDeployer.start(EJBDeployer.java:570) - locked <0x56679000> (a org.jboss.ejb.EJBDeployer) at org.jboss.deployment.MainDeployer.start(MainDeployer.java:836) at org.jboss.deployment.MainDeployer.start(MainDeployer.java:828) at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:645) at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:608) at sun.reflect.GeneratedMethodAccessor36.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:61) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:53) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473) at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176) at $Proxy7.deploy(Unknown Source) at org.jboss.deployment.scanner.URLDeploymentScanner.deploy(URLDeploymentScanner.java:304) at org.jboss.deployment.scanner.URLDeploymentScanner.scan(URLDeploymentScanner.java:478) - locked <0x56505080> (a org.jboss.deployment.scanner.URLDeploymentScanner) at org.jboss.deployment.scanner.AbstractDeploymentScanner$ScannerThread.doScan(AbstractDeploymentScanner.java:201) at org.jboss.deployment.scanner.AbstractDeploymentScanner.startService(AbstractDeploymentScanner.java:274) - locked <0x565044c0> (a org.jboss.deployment.scanner.AbstractDeploymentScanner$ScannerThread) at org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:271) at org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:221) at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:61) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:53) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473) at org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:837) at $Proxy0.start(Unknown Source) at org.jboss.system.ServiceController.start(ServiceController.java:367) - locked <0x56505440> (a org.jboss.system.ServiceController) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:61) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:53) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473) at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176) at $Proxy4.start(Unknown Source) at org.jboss.deployment.SARDeployer.start(SARDeployer.java:251) at org.jboss.deployment.MainDeployer.start(MainDeployer.java:836) at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:645) at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:608) at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:592) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:324) at org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:61) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:53) at org.jboss.mx.server.Invocation.invoke(Invocation.java:86) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473) at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176) at $Proxy5.deploy(Unknown Source) at org.jboss.system.server.ServerImpl.doStart(ServerImpl.java:407) at org.jboss.system.server.ServerImpl.start(ServerImpl.java:311) at org.jboss.Main.boot(Main.java:145) at org.jboss.Main$1.run(Main.java:399) at java.lang.Thread.run(Thread.java:534) Found 1 deadlock.