MessagingClusterHealthMBean may block when stopping a node ---------------------------------------------------------- Key: JBMESSAGING-1896 URL: https://issues.jboss.org/browse/JBMESSAGING-1896 Project: JBoss Messaging Issue Type: Bug Components: JMS Clustering Affects Versions: 1.4.8.SP3, 1.4.0.SP3.CP14 Reporter: Yong Hao Gao Assignee: Yong Hao Gao Fix For: 1.4.0.SP3.CP15, 1.4.8.SP4 When a node is shunned from the node but at the same time its DB is also disconnected, the MessagingClusterHealthMBean will shut down the node in the jgroups thread. Because the shutdown happens in the process of view change, it will block on the MessagingPostOffice.stopViewUpdate() method. I will wait for the flag updateInProcess forever because it will never get a chance to update the flag. See the stack trace: "Incoming-10,192.168.1.110:55200" prio=10 tid=0x00007f619c036800 nid=0x4b2e in Object.wait() [0x00007f618a24a000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000000d94c55d8> (a java.lang.Object) at java.lang.Object.wait(Object.java:502) at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.stopViewUpdate(MessagingPostOffice.java:852) - locked <0x00000000d94c55d8> (a java.lang.Object) at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.stop(MessagingPostOffice.java:796) at org.jboss.messaging.core.jmx.MessagingPostOfficeService.stopService(MessagingPostOfficeService.java:554) at org.jboss.system.ServiceMBeanSupport.jbossInternalStop(ServiceMBeanSupport.java:405) at org.jboss.system.ServiceMBeanSupport.stop(ServiceMBeanSupport.java:281) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:157) at org.jboss.mx.server.Invocation.dispatch(Invocation.java:96) at org.jboss.mx.interceptor.AbstractInterceptor.invoke(AbstractInterceptor.java:138) at org.jboss.mx.server.Invocation.invoke(Invocation.java:90) at org.jboss.mx.interceptor.ModelMBeanOperationInterceptor.invoke(ModelMBeanOperationInterceptor.java:140) at org.jboss.mx.server.Invocation.invoke(Invocation.java:90) at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264) at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:668) at org.jboss.jms.server.MessagingClusterHealthMBean.stopService(MessagingClusterHealthMBean.java:140) at org.jboss.jms.server.MessagingClusterHealthMBean.stopNodeOnDBFailure(MessagingClusterHealthMBean.java:107) - locked <0x00000000d731a520> (a org.jboss.jms.server.MessagingClusterHealthMBean) at org.jboss.jms.server.ServerPeer.stopJBMNodeForRecovery(ServerPeer.java:2115) at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.quarantine(MessagingPostOffice.java:1748) at org.jboss.messaging.core.impl.postoffice.MessagingPostOffice.nodesLeft(MessagingPostOffice.java:1568) at org.jboss.messaging.core.impl.postoffice.GroupMember$ControlMembershipListener.viewAccepted(GroupMember.java:609) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.handleUpEvent(MessageDispatcher.java:733) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:776) at org.jgroups.JChannel.up(JChannel.java:1336) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:454) at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:439) at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:153) at org.jgroups.protocols.FRAG2.up(FRAG2.java:188) at org.jgroups.protocols.FC.up(FC.java:493) at org.jgroups.protocols.pbcast.GMS.installView(GMS.java:543) - locked <0x00000000d94b2c90> (a org.jgroups.Membership) at org.jgroups.protocols.pbcast.CoordGmsImpl.handleViewChange(CoordGmsImpl.java:463) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:749) at org.jgroups.protocols.VIEW_SYNC.up(VIEW_SYNC.java:192) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233) at org.jgroups.protocols.UNICAST.up(UNICAST.java:328) at org.jgroups.protocols.pbcast.NAKACK.handleMessage(NAKACK.java:895) at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:708) at org.jgroups.protocols.BARRIER.up(BARRIER.java:136) at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167) at org.jgroups.protocols.FD.up(FD.java:284) at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:328) at org.jgroups.protocols.MERGE2.up(MERGE2.java:144) at org.jgroups.protocols.Discovery.up(Discovery.java:264) at org.jgroups.protocols.PING.up(PING.java:273) at org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2319) at org.jgroups.protocols.TP.passMessageUp(TP.java:1253) at org.jgroups.protocols.TP.access$100(TP.java:50) at org.jgroups.protocols.TP$1.run(TP.java:1177) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:636) -- This message is automatically generated by JIRA. If you think it was sent incorrectly, please contact your JIRA administrators: https://issues.jboss.org/secure/ContactAdministrators!default.jspa For more information on JIRA, see: http://www.atlassian.com/software/jira