[jboss-dev-forums] [Design of POJO Server] - Re: Strange classloading behavior -- thread stuck

Second set of stack traces taken today w/ mod_cluster 1.0.0.Beta3; not sure what AS version but I expect still 5.0.0.GA. First stack trace for problem thread: | "Incoming-2,192.168.1.5:42274" prio=10 tid=0x0000000040353800 nid=0x2b3a runnable [0x000000003a2fc000..0x000000003a2ffb70] | java.lang.Thread.State: RUNNABLE | at java.lang.Object.notify(Native Method) | at org.jboss.classloader.spi.base.ClassLoaderManager.unregisterLoaderThread(ClassLoaderManager.java:127) | - locked <0x000000001ce12608> (a java.util.Collections$SynchronizedList) | - locked <0x000000001ce12608> (a java.util.Collections$SynchronizedList) | at org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:1011) | - locked <0x00000000199bace8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:894) | at org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:289) | at org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) | at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) | at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) | - locked <0x000000001509f8e8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at java.lang.ClassLoader.loadClass(ClassLoader.java:252) | at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320) | - locked <0x000000001509f8e8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at java.lang.ClassLoader.defineClass1(Native Method) | at java.lang.ClassLoader.defineClass(ClassLoader.java:621) | at org.jboss.classloader.spi.base.BaseClassLoader.access$200(BaseClassLoader.java:62) | at org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:509) | at org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:469) | at java.security.AccessController.doPrivileged(Native Method) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:467) | - locked <0x000000001509f8e8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:444) | at org.jboss.classloader.spi.base.BaseDelegateLoader.loadClass(BaseDelegateLoader.java:134) | at org.jboss.classloader.spi.filter.FilteredDelegateLoader.loadClass(FilteredDelegateLoader.java:131) | at org.jboss.classloader.spi.base.ClassLoadingTask$ThreadTask.run(ClassLoadingTask.java:452) | at org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:254) | at org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) | at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) | at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) | - locked <0x000000001509f8e8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at java.lang.ClassLoader.loadClass(ClassLoader.java:252) | at org.jboss.invocation.MarshalledValueInputStream.resolveClass(MarshalledValueInputStream.java:101) | at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1575) | at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496) | at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1583) | at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496) | at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732) | at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) | at java.io.ObjectInputStream.readObject(ObjectInputStream.java:351) | at org.jboss.ha.framework.server.ClusterPartition.objectFromByteBufferResponseInternal(ClusterPartition.java:1887) | at org.jboss.ha.framework.server.ClusterPartition$ResponseMarshallerImpl.objectFromByteBuffer(ClusterPartition.java:2084) | at org.jgroups.blocks.RpcDispatcher$MarshallerAdapter.objectFromByteBuffer(RpcDispatcher.java:138) | at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:557) | at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:368) | at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:775) | at org.jgroups.JChannel.up(JChannel.java:1274) | at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462) | at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:377) | at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:144) | at org.jgroups.protocols.FRAG2.up(FRAG2.java:192) | at org.jgroups.protocols.FC.up(FC.java:468) | at org.jgroups.protocols.pbcast.GMS.up(GMS.java:796) | at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233) | at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616) | at org.jgroups.protocols.UNICAST.up(UNICAST.java:282) | at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:747) | at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167) | at org.jgroups.protocols.FD.up(FD.java:284) | at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:308) | at org.jgroups.protocols.MERGE2.up(MERGE2.java:144) | at org.jgroups.protocols.Discovery.up(Discovery.java:263) | at org.jgroups.protocols.PING.up(PING.java:270) | at org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2319) | at org.jgroups.protocols.TP.passMessageUp(TP.java:1253) | at org.jgroups.protocols.TP.access$100(TP.java:49) | at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1830) | at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1809) | at | java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | at java.lang.Thread.run(Thread.java:619) | Another thread Bela included in his problem report; not sure if/how it relates and whether it the trace was taken at same time as the one above or later, with the second version below. | "ajp-192.168.1.5-8009-2" daemon prio=10 tid=0x000000004575b800 nid=0x2b76 in Object.wait() [0x00000000028b6000..0x00000000028b9c70] | java.lang.Thread.State: WAITING (on object monitor) | at java.lang.Object.wait(Native Method) | - waiting on <0x000000001dbed570> (a java.util.Collections$SynchronizedList) | at java.lang.Object.wait(Object.java:485) | at org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:207) | - locked <0x000000001dbed570> (a java.util.Collections$SynchronizedList) | at org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) | at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) | at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) | - locked <0x0000000019928de8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at java.lang.ClassLoader.loadClass(ClassLoader.java:252) | at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320) | - locked <0x0000000019928de8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at org.apache.jasper.compiler.JspUtil.<clinit>(JspUtil.java:66) | at org.apache.jasper.JspCompilationContext.getServletClassName(JspCompilationContext.java:370) | at org.apache.jasper.JspCompilationContext.getClassFileName(JspCompilationContext.java:510) | at org.apache.jasper.compiler.Compiler.isOutDated(Compiler.java:421) | at org.apache.jasper.compiler.Compiler.isOutDated(Compiler.java:369) | at org.apache.jasper.JspCompilationContext.compile(JspCompilationContext.java:581) | at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:312) | - locked <0x000000001dbed890> (a org.apache.jasper.servlet.JspServletWrapper) | at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:322) | at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:249) | at javax.servlet.http.HttpServlet.service(HttpServlet.java:717) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96) | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) | at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:235) | at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191) | at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:190) | at org.jboss.web.tomcat.service.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:109) | at org.jboss.web.tomcat.service.session.JvmRouteValve.invoke(JvmRouteValve.java:87) | at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:92) | at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.process(SecurityContextEstablishmentValve.java:126) | at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.invoke(SecurityContextEstablishmentValve.java:70) | at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127) | at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) | at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) | at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:330) | at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:436) | at org.apache.coyote.ajp.AjpProtocol$AjpConnectionHandler.process(AjpProtocol.java:384) | at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447) | at java.lang.Thread.run(Thread.java:619) A second stack trace taken later for the JGroups "Incoming" thread shown above: | "Incoming-2,192.168.1.5:42274" prio=10 tid=0x0000000040353800 nid=0x2b3a runnable [0x000000003a2fc000..0x000000003a2ffb70] | java.lang.Thread.State: RUNNABLE | at java.lang.Object.notify(Native Method) | at org.jboss.classloader.spi.base.ClassLoaderManager.unregisterLoaderThread(ClassLoaderManager.java:127) | - locked <0x000000001ce12608> (a java.util.Collections$SynchronizedList) | - locked <0x000000001ce12608> (a java.util.Collections$SynchronizedList) | at org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:1011) | - locked <0x00000000199bace8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at org.jboss.classloader.spi.base.BaseClassLoader.unlock(BaseClassLoader.java:894) | at org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:289) | at org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) | at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) | at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) | - locked <0x000000001509f8e8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at java.lang.ClassLoader.loadClass(ClassLoader.java:252) | at java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320) | - locked <0x000000001509f8e8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at java.lang.ClassLoader.defineClass1(Native Method) | at java.lang.ClassLoader.defineClass(ClassLoader.java:621) | at org.jboss.classloader.spi.base.BaseClassLoader.access$200(BaseClassLoader.java:62) | at org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:509) | at org.jboss.classloader.spi.base.BaseClassLoader$2.run(BaseClassLoader.java:469) | at java.security.AccessController.doPrivileged(Native Method) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:467) | - locked <0x000000001509f8e8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClassLocally(BaseClassLoader.java:444) | at org.jboss.classloader.spi.base.BaseDelegateLoader.loadClass(BaseDelegateLoader.java:134) | at org.jboss.classloader.spi.filter.FilteredDelegateLoader.loadClass(FilteredDelegateLoader.java:131) | at org.jboss.classloader.spi.base.ClassLoadingTask$ThreadTask.run(ClassLoadingTask.java:452) | at org.jboss.classloader.spi.base.ClassLoaderManager.nextTask(ClassLoaderManager.java:254) | at org.jboss.classloader.spi.base.ClassLoaderManager.process(ClassLoaderManager.java:148) | at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:256) | at org.jboss.classloader.spi.base.BaseClassLoaderDomain.loadClass(BaseClassLoaderDomain.java:1099) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClassFromDomain(BaseClassLoader.java:735) | at org.jboss.classloader.spi.base.BaseClassLoader.loadClass(BaseClassLoader.java:378) | - locked <0x000000001509f8e8> (a org.jboss.classloader.spi.base.BaseClassLoader) | at java.lang.ClassLoader.loadClass(ClassLoader.java:252) | at org.jboss.invocation.MarshalledValueInputStream.resolveClass(MarshalledValueInputStream.java:101) | at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1575) | at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496) | at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1583) | at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496) | at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732) | at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329) | at java.io.ObjectInputStream.readObject(ObjectInputStream.java:351) | at org.jboss.ha.framework.server.ClusterPartition.objectFromByteBufferResponseInternal(ClusterPartition.java:1887) | at org.jboss.ha.framework.server.ClusterPartition$ResponseMarshallerImpl.objectFromByteBuffer(ClusterPartition.java:2084) | at org.jgroups.blocks.RpcDispatcher$MarshallerAdapter.objectFromByteBuffer(RpcDispatcher.java:138) | at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:557) | at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:368) | at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:775) | at org.jgroups.JChannel.up(JChannel.java:1274) | at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462) | at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:377) | at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:144) | at org.jgroups.protocols.FRAG2.up(FRAG2.java:192) | at org.jgroups.protocols.FC.up(FC.java:468) | at org.jgroups.protocols.pbcast.GMS.up(GMS.java:796) | at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233) | at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616) | at org.jgroups.protocols.UNICAST.up(UNICAST.java:282) | at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:747) | at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167) | at org.jgroups.protocols.FD.up(FD.java:284) | at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:308) | at org.jgroups.protocols.MERGE2.up(MERGE2.java:144) | at org.jgroups.protocols.Discovery.up(Discovery.java:263) | at org.jgroups.protocols.PING.up(PING.java:270) | at org.jgroups.protocols.TP$ProtocolAdapter.up(TP.java:2319) | at org.jgroups.protocols.TP.passMessageUp(TP.java:1253) | at org.jgroups.protocols.TP.access$100(TP.java:49) | at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1830) | at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1809) | at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | at java.lang.Thread.run(Thread.java:619) | Here the "Incoming" thread is caught at the same point in both dumps. Unlikely it's somehow stuck in java.lang.Object.notify() so seems likely to me that the thread's in a tight loop. View the original post : http://www.jboss.com/index.html?module=bb&op=viewtopic&p=4206336#... Reply to the post : http://www.jboss.com/index.html?module=bb&op=posting&mode=reply&a...