[jboss-jira] [JBoss JIRA] (AS7-3460) State transfer hangs, dominates MSC service start, and never returns

State transfer hangs, dominates MSC service start, and never returns -------------------------------------------------------------------- Key: AS7-3460 URL: https://issues.jboss.org/browse/AS7-3460 Project: Application Server 7 Issue Type: Bug Components: Clustering Reporter: Jason Greene Assignee: Paul Ferraro Priority: Critical Fix For: 7.1.0.Final Now: "MSC service thread 1-3" prio=10 tid=0x00007fb118001800 nid=0x7f04 in Object.wait() [0x00007fb154bf5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000000e1b36470> (a java.lang.Object) at java.lang.Object.wait(Object.java:485) at org.infinispan.statetransfer.StateTransferLockImpl.acquireLockForCommitCommand(StateTransferLockImpl.java:369) - locked <0x00000000e1b36470> (a java.lang.Object) at org.infinispan.statetransfer.StateTransferLockImpl.acquireForCommand(StateTransferLockImpl.java:144) at org.infinispan.interceptors.StateTransferLockInterceptor.visitCommitCommand(StateTransferLockInterceptor.java:115) at org.infinispan.commands.tx.CommitCommand.acceptVisitor(CommitCommand.java:66) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitCommitCommand(AbstractVisitor.java:121) at org.infinispan.commands.tx.CommitCommand.acceptVisitor(CommitCommand.java:66) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:130) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:89) at org.infinispan.commands.AbstractVisitor.visitCommitCommand(AbstractVisitor.java:121) at org.infinispan.commands.tx.CommitCommand.acceptVisitor(CommitCommand.java:66) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:130) at org.infinispan.commands.AbstractVisitor.visitCommitCommand(AbstractVisitor.java:121) at org.infinispan.commands.tx.CommitCommand.acceptVisitor(CommitCommand.java:66) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:116) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:86) at org.infinispan.commands.AbstractVisitor.visitCommitCommand(AbstractVisitor.java:121) at org.infinispan.commands.tx.CommitCommand.acceptVisitor(CommitCommand.java:66) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:345) at org.infinispan.transaction.TransactionCoordinator.commit(TransactionCoordinator.java:181) at org.infinispan.transaction.synchronization.SynchronizationAdapter.afterCompletion(SynchronizationAdapter.java:83) at org.infinispan.transaction.tm.DummyTransaction.notifyAfterCompletion(DummyTransaction.java:272) at org.infinispan.transaction.tm.DummyTransaction.runCommitTx(DummyTransaction.java:321) at org.infinispan.transaction.tm.DummyTransaction.commit(DummyTransaction.java:90) at org.infinispan.transaction.tm.DummyBaseTransactionManager.commit(DummyBaseTransactionManager.java:100) at org.infinispan.batch.BatchContainer.resolveTransaction(BatchContainer.java:123) at org.infinispan.batch.BatchContainer.endBatch(BatchContainer.java:105) at org.infinispan.batch.BatchContainer.endBatch(BatchContainer.java:86) at org.infinispan.CacheImpl.endBatch(CacheImpl.java:577) at org.infinispan.AbstractDelegatingCache.endBatch(AbstractDelegatingCache.java:75) at org.jboss.as.clustering.infinispan.invoker.BatchOperation.invoke(BatchOperation.java:26) at org.jboss.as.clustering.registry.RegistryService.invoke(RegistryService.java:260) at org.jboss.as.clustering.registry.RegistryService.stop(RegistryService.java:178) at org.jboss.msc.service.ServiceControllerImpl$StopTask.stopService(ServiceControllerImpl.java:1909) at org.jboss.msc.service.ServiceControllerImpl$StopTask.run(ServiceControllerImpl.java:1872) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) was: "MSC service thread 1-6" prio=10 tid=0x00007f207c038000 nid=0x23be waiting on condition [0x00007f209d475000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e1539140> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2116) at org.jgroups.util.Promise.doWait(Promise.java:116) at org.jgroups.util.Promise._getResultWithTimeout(Promise.java:72) at org.jgroups.util.Promise.getResultWithTimeout(Promise.java:41) at org.jgroups.util.Promise.getResult(Promise.java:103) at org.jgroups.protocols.pbcast.ClientGmsImpl.joinInternal(ClientGmsImpl.java:137) at org.jgroups.protocols.pbcast.ClientGmsImpl.joinWithStateTransfer(ClientGmsImpl.java:42) at org.jgroups.protocols.pbcast.GMS.down(GMS.java:921) at org.jgroups.protocols.FlowControl.down(FlowControl.java:351) at org.jgroups.protocols.FlowControl.down(FlowControl.java:351) at org.jgroups.protocols.FRAG2.down(FRAG2.java:147) at org.jgroups.protocols.pbcast.STATE_TRANSFER.down(STATE_TRANSFER.java:238) at org.jgroups.protocols.pbcast.FLUSH.handleConnect(FLUSH.java:300) at org.jgroups.protocols.pbcast.FLUSH.down(FLUSH.java:265) at org.jgroups.stack.ProtocolStack.down(ProtocolStack.java:1033) at org.jgroups.JChannel.down(JChannel.java:732) at org.jgroups.JChannel.connect(JChannel.java:347) - locked <0x00000000e1537880> (a org.jboss.as.clustering.jgroups.MuxChannel) at org.jgroups.JChannel.connect(JChannel.java:305) - locked <0x00000000e1537880> (a org.jboss.as.clustering.jgroups.MuxChannel) at org.jboss.as.clustering.jgroups.subsystem.ChannelService.start(ChannelService.java:38) at org.jboss.msc.service.ServiceControllerImpl$StartTask.startService(ServiceControllerImpl.java:1824) at org.jboss.msc.service.ServiceControllerImpl$StartTask.run(ServiceControllerImpl.java:1759) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) Log: 12:46:12,235 WARNING [org.jgroups.protocols.UDP] (MSC service thread 1-6) null: no physical address for node-udp-0/ejb, dropping message 12:46:15,240 WARNING [org.jgroups.protocols.pbcast.GMS] (MSC service thread 1-6) JOIN(node-udp-0/ejb) sent to node-udp-0/ejb timed out (after 3000 ms), retrying 12:46:17,242 WARNING [org.jgroups.protocols.UDP] (MSC service thread 1-6) null: no physical address for node-udp-0/ejb, dropping message 12:46:20,244 WARNING [org.jgroups.protocols.pbcast.GMS] (MSC service thread 1-6) JOIN(node-udp-0/ejb) sent to node-udp-0/ejb timed out (after 3000 ms), retrying 12:46:22,248 WARNING [org.jgroups.protocols.UDP] (MSC service thread 1-6) null: no physical address for node-udp-0/ejb, dropping message 12:46:25,252 WARNING [org.jgroups.protocols.pbcast.GMS] (MSC service thread 1-6) JOIN(node-udp-0/ejb) sent to node-udp-0/ejb timed out (after 3000 ms), retrying 12:46:27,256 WARNING [org.jgroups.protocols.UDP] (MSC service thread 1-6) null: no physical address for node-udp-0/ejb, dropping message 12:46:30,261 WARNING [org.jgroups.protocols.pbcast.GMS] (MSC service thread 1-6) JOIN(node-udp-0/ejb) sent to node-udp-0/ejb timed out (after 3000 ms), retrying 12:46:32,263 WARNING [org.jgroups.protocols.UDP] (MSC service thread 1-6) null: no physical address for node-udp-0/ejb, dropping message 12:46:35,266 WARNING [org.jgroups.protocols.pbcast.GMS] (MSC service thread 1-6) JOIN(node-udp-0/ejb) sent to node-udp-0/ejb timed out (after 3000 ms), retrying 12:46:37,268 WARNING [org.jgroups.protocols.UDP] (MSC service thread 1-6) null: no physical address for node-udp-0/ejb, dropping message 12:46:40,269 WARNING [org.jgroups.protocols.pbcast.GMS] (MSC service thread 1-6) JOIN(node-udp-0/ejb) sent to node-udp-0/ejb timed out (after 3000 ms), retrying 12:46:42,270 WARNING [org.jgroups.protocols.UDP] (MSC service thread 1-6) null: no physical address for node-udp-0/ejb, dropping message (repeats)