Second set of stack traces taken today w/ mod_cluster 1.0.0.Beta3; not sure what AS
version but I expect still 5.0.0.GA.
First stack trace for problem thread:
| "Incoming-2,192.168.1.5:42274" prio=10 tid=0x0000000040353800 nid=0x2b3a
runnable [0x000000003a2fc000..0x000000003a2ffb70]
| java.lang.Thread.State: RUNNABLE
| at java.lang.Object.notify(Native Method)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.unregisterLoaderThread(ClassLoaderManager.java:127)
| - locked <0x000000001ce12608> (a
java.util.Collections$SynchronizedList)
| - locked <0x000000001ce12608> (a
java.util.Collections$SynchronizedList)
| at
org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:1011)
| - locked <0x00000000199bace8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at
org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:894)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:289)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148)
| at
org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256)
| at
org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378)
| - locked <0x000000001509f8e8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at java.lang.ClassLoader.loadClass(ClassLoader.java:252)
| at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320)
| - locked <0x000000001509f8e8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at java.lang.ClassLoader.defineClass1(Native Method)
| at java.lang.ClassLoader.defineClass(ClassLoader.java:621)
| at
org.jboss.classloader.spi.base.BaseClassLoader.access$200(BaseClassLoader.java:62)
| at
org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:509)
| at
org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:469)
| at java.security.AccessController.doPrivileged(Native Method)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:467)
| - locked <0x000000001509f8e8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:444)
| at
org.jboss.classloader.spi.base.BaseDelegateLoader.loadClass(BaseDelegateLoader.java:134)
| at
org.jboss.classloader.spi.filter.FilteredDelegateLoader.loadClass(FilteredDelegateLoader.java:131)
| at
org.jboss.classloader.spi.base.ClassLoadingTask$ThreadTask.run(ClassLoadingTask.java:452)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:254)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148)
| at
org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256)
| at
org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378)
| - locked <0x000000001509f8e8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at java.lang.ClassLoader.loadClass(ClassLoader.java:252)
| at
org.jboss.invocation.MarshalledValueInputStream.resolveClass(MarshalledValueInputStream.java:101)
| at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1575)
| at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496)
| at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1583)
| at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496)
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732)
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:351)
| at
org.jboss.ha.framework.server.ClusterPartition.objectFromByteBufferResponseInternal(ClusterPartition.java:1887)
| at
org.jboss.ha.framework.server.ClusterPartition$ResponseMarshallerImpl.objectFromByteBuffer(ClusterPartition.java:2084)
| at
org.jgroups.blocks.RpcDispatcher$MarshallerAdapter.objectFromByteBuffer(RpcDispatcher.java:138)
| at
org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:557)
| at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:368)
| at
org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:775)
| at org.jgroups.JChannel.up(JChannel.java:1274)
| at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462)
| at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:377)
| at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:144)
| at org.jgroups.protocols.FRAG2.up(FRAG2.java:192)
| at org.jgroups.protocols.FC.up(FC.java:468)
| at org.jgroups.protocols.pbcast.GMS.up(GMS.java:796)
| at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233)
| at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616)
| at org.jgroups.protocols.UNICAST.up(UNICAST.java:282)
| at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:747)
| at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167)
| at org.jgroups.protocols.FD.up(FD.java:284)
| at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:308)
| at org.jgroups.protocols.MERGE2.up(MERGE2.java:144)
| at org.jgroups.protocols.Discovery.up(Discovery.java:263)
| at org.jgroups.protocols.PING.up(PING.java:270)
| at org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2319)
| at org.jgroups.protocols.TP.passMessageUp(TP.java:1253)
| at org.jgroups.protocols.TP.access$100(TP.java:49)
| at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1830)
| at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1809)
| at
|
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
| at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
| at java.lang.Thread.run(Thread.java:619)
|
Another thread Bela included in his problem report; not sure if/how it relates and whether
it the trace was taken at same time as the one above or later, with the second version
below.
| "ajp-192.168.1.5-8009-2" daemon prio=10 tid=0x000000004575b800 nid=0x2b76 in
Object.wait() [0x00000000028b6000..0x00000000028b9c70]
| java.lang.Thread.State: WAITING (on object monitor)
| at java.lang.Object.wait(Native Method)
| - waiting on <0x000000001dbed570> (a
java.util.Collections$SynchronizedList)
| at java.lang.Object.wait(Object.java:485)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:207)
| - locked <0x000000001dbed570> (a
java.util.Collections$SynchronizedList)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148)
| at
org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256)
| at
org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378)
| - locked <0x0000000019928de8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at java.lang.ClassLoader.loadClass(ClassLoader.java:252)
| at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320)
| - locked <0x0000000019928de8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at org.apache.jasper.compiler.JspUtil.<clinit>(JspUtil.java:66)
| at
org.apache.jasper.JspCompilationContext.getServletClassName(JspCompilationContext.java:370)
| at
org.apache.jasper.JspCompilationContext.getClassFileName(JspCompilationContext.java:510)
| at org.apache.jasper.compiler.Compiler.isOutDated(Compiler.java:421)
| at org.apache.jasper.compiler.Compiler.isOutDated(Compiler.java:369)
| at
org.apache.jasper.JspCompilationContext.compile(JspCompilationContext.java:581)
| at
org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:312)
| - locked <0x000000001dbed890> (a
org.apache.jasper.servlet.JspServletWrapper)
| at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:322)
| at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:249)
| at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
| at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
| at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
| at
org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96)
| at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
| at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
| at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:235)
| at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
| at
org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:190)
| at
org.jboss.web.tomcat.service.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:109)
| at
org.jboss.web.tomcat.service.session.JvmRouteValve.invoke(JvmRouteValve.java:87)
| at
org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:92)
| at
org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.process(SecurityContextEstablishmentValve.java:126)
| at
org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.invoke(SecurityContextEstablishmentValve.java:70)
| at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
| at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
| at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
| at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:330)
| at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:436)
| at
org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:384)
| at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447)
| at java.lang.Thread.run(Thread.java:619)
A second stack trace taken later for the JGroups "Incoming" thread shown above:
| "Incoming-2,192.168.1.5:42274" prio=10 tid=0x0000000040353800 nid=0x2b3a
runnable [0x000000003a2fc000..0x000000003a2ffb70]
| java.lang.Thread.State: RUNNABLE
| at java.lang.Object.notify(Native Method)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.unregisterLoaderThread(ClassLoaderManager.java:127)
| - locked <0x000000001ce12608> (a
java.util.Collections$SynchronizedList)
| - locked <0x000000001ce12608> (a
java.util.Collections$SynchronizedList)
| at
org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:1011)
| - locked <0x00000000199bace8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at
org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:894)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:289)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148)
| at
org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256)
| at
org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378)
| - locked <0x000000001509f8e8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at java.lang.ClassLoader.loadClass(ClassLoader.java:252)
| at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320)
| - locked <0x000000001509f8e8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at java.lang.ClassLoader.defineClass1(Native Method)
| at java.lang.ClassLoader.defineClass(ClassLoader.java:621)
| at
org.jboss.classloader.spi.base.BaseClassLoader.access$200(BaseClassLoader.java:62)
| at
org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:509)
| at
org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:469)
| at java.security.AccessController.doPrivileged(Native Method)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:467)
| - locked <0x000000001509f8e8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:444)
| at
org.jboss.classloader.spi.base.BaseDelegateLoader.loadClass(BaseDelegateLoader.java:134)
| at
org.jboss.classloader.spi.filter.FilteredDelegateLoader.loadClass(FilteredDelegateLoader.java:131)
| at
org.jboss.classloader.spi.base.ClassLoadingTask$ThreadTask.run(ClassLoadingTask.java:452)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:254)
| at
org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148)
| at
org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256)
| at
org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735)
| at
org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378)
| - locked <0x000000001509f8e8> (a
org.jboss.classloader.spi.base.BaseClassLoader)
| at java.lang.ClassLoader.loadClass(ClassLoader.java:252)
| at
org.jboss.invocation.MarshalledValueInputStream.resolveClass(MarshalledValueInputStream.java:101)
| at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1575)
| at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496)
| at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1583)
| at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496)
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732)
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:351)
| at
org.jboss.ha.framework.server.ClusterPartition.objectFromByteBufferResponseInternal(ClusterPartition.java:1887)
| at
org.jboss.ha.framework.server.ClusterPartition$ResponseMarshallerImpl.objectFromByteBuffer(ClusterPartition.java:2084)
| at
org.jgroups.blocks.RpcDispatcher$MarshallerAdapter.objectFromByteBuffer(RpcDispatcher.java:138)
| at
org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:557)
| at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:368)
| at
org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:775)
| at org.jgroups.JChannel.up(JChannel.java:1274)
| at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462)
| at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:377)
| at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:144)
| at org.jgroups.protocols.FRAG2.up(FRAG2.java:192)
| at org.jgroups.protocols.FC.up(FC.java:468)
| at org.jgroups.protocols.pbcast.GMS.up(GMS.java:796)
| at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233)
| at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616)
| at org.jgroups.protocols.UNICAST.up(UNICAST.java:282)
| at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:747)
| at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167)
| at org.jgroups.protocols.FD.up(FD.java:284)
| at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:308)
| at org.jgroups.protocols.MERGE2.up(MERGE2.java:144)
| at org.jgroups.protocols.Discovery.up(Discovery.java:263)
| at org.jgroups.protocols.PING.up(PING.java:270)
| at org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2319)
| at org.jgroups.protocols.TP.passMessageUp(TP.java:1253)
| at org.jgroups.protocols.TP.access$100(TP.java:49)
| at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1830)
| at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1809)
| at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
| at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
| at java.lang.Thread.run(Thread.java:619)
|
Here the "Incoming" thread is caught at the same point in both dumps. Unlikely
it's somehow stuck in java.lang.Object.notify() so seems likely to me that the
thread's in a tight loop.
View the original post :
http://www.jboss.com/index.html?module=bb&op=viewtopic&p=4206336#...
Reply to the post :
http://www.jboss.com/index.html?module=bb&op=posting&mode=reply&a...