Trying to access a webapp (/web) in the node1/node2 cluster: the browser hangs and eventually (after several minutes returns a 503). CTRL-3 on node1 yielded several long stack traces (this is 2 dumps). Can someone look into it ? This goes on forever, even after the 503 is returned... Name: Incoming-12,192.168.1.5:57544 State: RUNNABLE Total blocked: 4 Total waited: 24,105 Stack trace: java.util.Collections$SynchronizedList.remove(Collections.java:1825) - locked java.util.Collections$SynchronizedList@45b84e09 org.jboss.classloader.spi.base.ClassLoaderManager.unregisterLoaderThread(ClassLoaderManager.java:115) - locked java.util.Collections$SynchronizedList@45b84e09 org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:1011) - locked org.jboss.classloader.spi.base.BaseClassLoader@7b17ad7b org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:894) org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:289) org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) - locked org.jboss.classloader.spi.base.BaseClassLoader@3bbe9a50 java.lang.ClassLoader.loadClass(ClassLoader.java:252) java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320) - locked org.jboss.classloader.spi.base.BaseClassLoader@3bbe9a50 java.lang.ClassLoader.defineClass1(Native Method) java.lang.ClassLoader.defineClass(ClassLoader.java:621) org.jboss.classloader.spi.base.BaseClassLoader.access$200(BaseClassLoader.java:62) org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:509) org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:469) java.security.AccessController.doPrivileged(Native Method) org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:467) - locked org.jboss.classloader.spi.base.BaseClassLoader@3bbe9a50 org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:444) org.jboss.classloader.spi.base.BaseDelegateLoader.loadClass(BaseDelegateLoader.java:134) org.jboss.classloader.spi.filter.FilteredDelegateLoader.loadClass(FilteredDelegateLoader.java:131) org.jboss.classloader.spi.base.ClassLoadingTask$ThreadTask.run(ClassLoadingTask.java:452) org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:254) org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) - locked org.jboss.classloader.spi.base.BaseClassLoader@3bbe9a50 java.lang.ClassLoader.loadClass(ClassLoader.java:252) org.jboss.invocation.MarshalledValueInputStream.resolveClass(MarshalledValueInputStream.java:101) java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1575) java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496) java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732) java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) java.io.ObjectInputStream.readObject(ObjectInputStream.java:351) java.util.HashSet.readObject(HashSet.java:291) sun.reflect.GeneratedMethodAccessor209.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) java.lang.reflect.Method.invoke(Method.java:597) java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:974) java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1849) java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753) java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1947) java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1871) java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753) java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) java.io.ObjectInputStream.readObject(ObjectInputStream.java:351) org.jboss.ha.framework.server.ClusterPartition.objectFromByteBufferResponseInternal(ClusterPartition.java:1887) org.jboss.ha.framework.server.ClusterPartition$ResponseMarshallerImpl.objectFromByteBuffer(ClusterPartition.java:2084) org.jgroups.blocks.RpcDispatcher$MarshallerAdapter.objectFromByteBuffer(RpcDispatcher.java:138) org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:557) org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:368) org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:775) org.jgroups.JChannel.up(JChannel.java:1274) org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462) org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:377) org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:144) org.jgroups.protocols.FRAG2.up(FRAG2.java:192) org.jgroups.protocols.FC.up(FC.java:468) org.jgroups.protocols.pbcast.GMS.up(GMS.java:796) org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233) org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616) org.jgroups.protocols.UNICAST.up(UNICAST.java:282) org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:747) org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167) org.jgroups.protocols.FD.up(FD.java:284) org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:308) org.jgroups.protocols.MERGE2.up(MERGE2.java:144) org.jgroups.protocols.Discovery.up(Discovery.java:263) org.jgroups.protocols.PING.up(PING.java:270) org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2319) org.jgroups.protocols.TP.passMessageUp(TP.java:1253) org.jgroups.protocols.TP.access$100(TP.java:49) org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1830) org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1809) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) "Incoming-12,192.168.1.5:57544" prio=10 tid=0x00000000455b1000 nid=0x7b55 runnable [0x000000003ed7f000..0x000000003ed82bf0] java.lang.Thread.State: RUNNABLE at java.util.Collections$SynchronizedCollection.isEmpty(Collections.java:1560) - locked <0x000000001d3ce1d0> (a java.util.Collections$SynchronizedList) at org.jboss.classloader.spi.base.ClassLoaderManager.unregisterLoaderThread(ClassLoaderManager.java:113) - locked <0x000000001d3ce1d0> (a java.util.Collections$SynchronizedList) at org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:1011) - locked <0x000000001ac6f480> (a org.jboss.classloader.spi.base.BaseClassLoader) at org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:894) at org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:289) at org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) at org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) at org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) - locked <0x0000000015c73390> (a org.jboss.classloader.spi.base.BaseClassLoader) at java.lang.ClassLoader.loadClass(ClassLoader.java:252) at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320) - locked <0x0000000015c73390> (a org.jboss.classloader.spi.base.BaseClassLoader) at java.lang.ClassLoader.defineClass1(Native Method) at java.lang.ClassLoader.defineClass(ClassLoader.java:621) at org.jboss.classloader.spi.base.BaseClassLoader.access$200(BaseClassLoader.java:62) at org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:509) at org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:469) at java.security.AccessController.doPrivileged(Native Method) at org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:467) - locked <0x0000000015c73390> (a org.jboss.classloader.spi.base.BaseClassLoader) at org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:444) at org.jboss.classloader.spi.base.BaseDelegateLoader.loadClass(BaseDelegateLoader.java:134) at org.jboss.classloader.spi.filter.FilteredDelegateLoader.loadClass(FilteredDelegateLoader.java:131) at org.jboss.classloader.spi.base.ClassLoadingTask$ThreadTask.run(ClassLoadingTask.java:452) at org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:254) at org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) at org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) at org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) - locked <0x0000000015c73390> (a org.jboss.classloader.spi.base.BaseClassLoader) at java.lang.ClassLoader.loadClass(ClassLoader.java:252) at org.jboss.invocation.MarshalledValueInputStream.resolveClass(MarshalledValueInputStream.java:101) at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1575) at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) at java.io.ObjectInputStream.readObject(ObjectInputStream.java:351) at java.util.HashSet.readObject(HashSet.java:291) at sun.reflect.GeneratedMethodAccessor209.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:974) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1849) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1947) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1871) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) at java.io.ObjectInputStream.readObject(ObjectInputStream.java:351) at org.jboss.ha.framework.server.ClusterPartition.objectFromByteBufferResponseInternal(ClusterPartition.java:1887) at org.jboss.ha.framework.server.ClusterPartition$ResponseMarshallerImpl.objectFromByteBuffer(ClusterPartition.java:2084) at org.jgroups.blocks.RpcDispatcher$MarshallerAdapter.objectFromByteBuffer(RpcDispatcher.java:138) at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:557) at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:368) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:775) at org.jgroups.JChannel.up(JChannel.java:1274) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462) at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:377) at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:144) at org.jgroups.protocols.FRAG2.up(FRAG2.java:192) at org.jgroups.protocols.FC.up(FC.java:468) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:796) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233) at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616) at org.jgroups.protocols.UNICAST.up(UNICAST.java:282) at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:747) at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167) at org.jgroups.protocols.FD.up(FD.java:284) at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:308) at org.jgroups.protocols.MERGE2.up(MERGE2.java:144) at org.jgroups.protocols.Discovery.up(Discovery.java:263) at org.jgroups.protocols.PING.up(PING.java:270) at org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2319) at org.jgroups.protocols.TP.passMessageUp(TP.java:1253) at org.jgroups.protocols.TP.access$100(TP.java:49) at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1830) at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1809) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619)

Brian Stansberry wrote: > A few questions: > > 1) Is this with a stock AS 5.0.0.GA and a standard install of > mod_cluster? I think yes, but am double-checking? Yes > 2) Is this reproducible? Yes. If you run https://jira.jboss.org/jira/browse/MODCLUSTER-31 (IIRC), you always have these long stack traces.

> 3) Any other logging? > > This seems more like a problem in the AS classloading layer which > we'll have to sort out. It's odd too that this led to a browser hang > and a 503. The mod_cluster code shouldn't be calling into JGroups as > part of request handling; all the JGroups stuff is background work. mod-cluster doesn't call into JGroups, in both cases JGroups received a message and ClusterPartition is unmarshalling it

> Bela Ban wrote: >> Trying to access a webapp (/web) in the node1/node2 cluster: the >> browser hangs and eventually (after several minutes returns a 503). >> CTRL-3 on node1 yielded several long stack traces (this is 2 dumps). >> Can someone look into it ? This goes on forever, even after the 503 >> is returned... >> >> >> >> Name: Incoming-12,192.168.1.5:57544 >> State: RUNNABLE >> Total blocked: 4 Total waited: 24,105 >> >> Stack trace: >> java.util.Collections$SynchronizedList.remove(Collections.java:1825) >> - locked java.util.Collections$SynchronizedList@45b84e09 >> org.jboss.classloader.spi.base.ClassLoaderManager.unregisterLoaderThread(ClassLoaderManager.java:115) >> >> - locked java.util.Collections$SynchronizedList@45b84e09 >> org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:1011) >> >> - locked org.jboss.classloader.spi.base.BaseClassLoader@7b17ad7b >> org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:894) >> >> org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:289) >> >> org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) >> >> org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) >> >> org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) >> >> org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) >> >> org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) >> >> - locked org.jboss.classloader.spi.base.BaseClassLoader@3bbe9a50 >> java.lang.ClassLoader.loadClass(ClassLoader.java:252) >> java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320) >> - locked org.jboss.classloader.spi.base.BaseClassLoader@3bbe9a50 >> java.lang.ClassLoader.defineClass1(Native Method) >> java.lang.ClassLoader.defineClass(ClassLoader.java:621) >> org.jboss.classloader.spi.base.BaseClassLoader.access$200(BaseClassLoader.java:62) >> >> org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:509) >> >> org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:469) >> >> java.security.AccessController.doPrivileged(Native Method) >> org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:467) >> >> - locked org.jboss.classloader.spi.base.BaseClassLoader@3bbe9a50 >> org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:444) >> >> org.jboss.classloader.spi.base.BaseDelegateLoader.loadClass(BaseDelegateLoader.java:134) >> >> org.jboss.classloader.spi.filter.FilteredDelegateLoader.loadClass(FilteredDelegateLoader.java:131) >> >> org.jboss.classloader.spi.base.ClassLoadingTask$ThreadTask.run(ClassLoadingTask.java:452) >> >> org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:254) >> >> org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) >> >> org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) >> >> org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) >> >> org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) >> >> org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) >> >> - locked org.jboss.classloader.spi.base.BaseClassLoader@3bbe9a50 >> java.lang.ClassLoader.loadClass(ClassLoader.java:252) >> org.jboss.invocation.MarshalledValueInputStream.resolveClass(MarshalledValueInputStream.java:101) >> >> java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1575) >> java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496) >> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732) >> >> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) >> java.io.ObjectInputStream.readObject(ObjectInputStream.java:351) >> java.util.HashSet.readObject(HashSet.java:291) >> sun.reflect.GeneratedMethodAccessor209.invoke(Unknown Source) >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) >> >> java.lang.reflect.Method.invoke(Method.java:597) >> java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:974) >> java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1849) >> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753) >> >> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) >> java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1947) >> java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1871) >> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753) >> >> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) >> java.io.ObjectInputStream.readObject(ObjectInputStream.java:351) >> org.jboss.ha.framework.server.ClusterPartition.objectFromByteBufferResponseInternal(ClusterPartition.java:1887) >> >> org.jboss.ha.framework.server.ClusterPartition$ResponseMarshallerImpl.objectFromByteBuffer(ClusterPartition.java:2084) >> >> org.jgroups.blocks.RpcDispatcher$MarshallerAdapter.objectFromByteBuffer(RpcDispatcher.java:138) >> >> org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:557) >> >> org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:368) >> org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:775) >> >> org.jgroups.JChannel.up(JChannel.java:1274) >> org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462) >> org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:377) >> org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:144) >> org.jgroups.protocols.FRAG2.up(FRAG2.java:192) >> org.jgroups.protocols.FC.up(FC.java:468) >> org.jgroups.protocols.pbcast.GMS.up(GMS.java:796) >> org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233) >> org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616) >> org.jgroups.protocols.UNICAST.up(UNICAST.java:282) >> org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:747) >> org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167) >> org.jgroups.protocols.FD.up(FD.java:284) >> org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:308) >> org.jgroups.protocols.MERGE2.up(MERGE2.java:144) >> org.jgroups.protocols.Discovery.up(Discovery.java:263) >> org.jgroups.protocols.PING.up(PING.java:270) >> org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2319) >> org.jgroups.protocols.TP.passMessageUp(TP.java:1253) >> org.jgroups.protocols.TP.access$100(TP.java:49) >> org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1830) >> org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1809) >> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) >> >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) >> >> java.lang.Thread.run(Thread.java:619) >> >> >> >> >> "Incoming-12,192.168.1.5:57544" prio=10 tid=0x00000000455b1000 >> nid=0x7b55 runnable [0x000000003ed7f000..0x000000003ed82bf0] >> java.lang.Thread.State: RUNNABLE >> at >> java.util.Collections$SynchronizedCollection.isEmpty(Collections.java:1560) >> >> - locked <0x000000001d3ce1d0> (a >> java.util.Collections$SynchronizedList) >> at >> org.jboss.classloader.spi.base.ClassLoaderManager.unregisterLoaderThread(ClassLoaderManager.java:113) >> >> - locked <0x000000001d3ce1d0> (a >> java.util.Collections$SynchronizedList) >> at >> org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:1011) >> >> - locked <0x000000001ac6f480> (a >> org.jboss.classloader.spi.base.BaseClassLoader) >> at >> org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:894) >> >> at >> org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:289) >> >> at >> org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) >> >> at >> org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) >> >> at >> org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) >> >> at >> org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) >> >> at >> org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) >> >> - locked <0x0000000015c73390> (a >> org.jboss.classloader.spi.base.BaseClassLoader) >> at java.lang.ClassLoader.loadClass(ClassLoader.java:252) >> at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320) >> - locked <0x0000000015c73390> (a >> org.jboss.classloader.spi.base.BaseClassLoader) >> at java.lang.ClassLoader.defineClass1(Native Method) >> at java.lang.ClassLoader.defineClass(ClassLoader.java:621) >> at >> org.jboss.classloader.spi.base.BaseClassLoader.access$200(BaseClassLoader.java:62) >> >> at >> org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:509) >> >> at >> org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:469) >> >> at java.security.AccessController.doPrivileged(Native Method) >> at >> org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:467) >> >> - locked <0x0000000015c73390> (a >> org.jboss.classloader.spi.base.BaseClassLoader) >> at >> org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:444) >> >> at >> org.jboss.classloader.spi.base.BaseDelegateLoader.loadClass(BaseDelegateLoader.java:134) >> >> at >> org.jboss.classloader.spi.filter.FilteredDelegateLoader.loadClass(FilteredDelegateLoader.java:131) >> >> at >> org.jboss.classloader.spi.base.ClassLoadingTask$ThreadTask.run(ClassLoadingTask.java:452) >> >> at >> org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:254) >> >> at >> org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) >> >> at >> org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) >> >> at >> org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) >> >> at >> org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) >> >> at >> org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) >> >> - locked <0x0000000015c73390> (a >> org.jboss.classloader.spi.base.BaseClassLoader) >> at java.lang.ClassLoader.loadClass(ClassLoader.java:252) >> at >> org.jboss.invocation.MarshalledValueInputStream.resolveClass(MarshalledValueInputStream.java:101) >> >> at >> java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1575) >> at >> java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496) >> at >> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732) >> >> at >> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) >> at >> java.io.ObjectInputStream.readObject(ObjectInputStream.java:351) >> at java.util.HashSet.readObject(HashSet.java:291) >> at sun.reflect.GeneratedMethodAccessor209.invoke(Unknown Source) >> at >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) >> >> at java.lang.reflect.Method.invoke(Method.java:597) >> at >> java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:974) >> at >> java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1849) >> at >> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753) >> >> at >> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) >> at >> java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1947) >> at >> java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1871) >> at >> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753) >> >> at >> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) >> at >> java.io.ObjectInputStream.readObject(ObjectInputStream.java:351) >> at >> org.jboss.ha.framework.server.ClusterPartition.objectFromByteBufferResponseInternal(ClusterPartition.java:1887) >> >> at >> org.jboss.ha.framework.server.ClusterPartition$ResponseMarshallerImpl.objectFromByteBuffer(ClusterPartition.java:2084) >> >> at >> org.jgroups.blocks.RpcDispatcher$MarshallerAdapter.objectFromByteBuffer(RpcDispatcher.java:138) >> >> at >> org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:557) >> >> at >> org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:368) >> at >> org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:775) >> >> at org.jgroups.JChannel.up(JChannel.java:1274) >> at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462) >> at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:377) >> at >> org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:144) >> at org.jgroups.protocols.FRAG2.up(FRAG2.java:192) >> at org.jgroups.protocols.FC.up(FC.java:468) >> at org.jgroups.protocols.pbcast.GMS.up(GMS.java:796) >> at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233) >> at >> org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616) >> at org.jgroups.protocols.UNICAST.up(UNICAST.java:282) >> at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:747) >> at >> org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167) >> at org.jgroups.protocols.FD.up(FD.java:284) >> at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:308) >> at org.jgroups.protocols.MERGE2.up(MERGE2.java:144) >> at org.jgroups.protocols.Discovery.up(Discovery.java:263) >> at org.jgroups.protocols.PING.up(PING.java:270) >> at org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2319) >> at org.jgroups.protocols.TP.passMessageUp(TP.java:1253) >> at org.jgroups.protocols.TP.access$100(TP.java:49) >> at >> org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1830) >> at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1809) >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) >> >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) >> >> at java.lang.Thread.run(Thread.java:619) >> > >