[jboss-jira] [JBoss JIRA] Updated: (JBCACHE-1103) Make cache loader implementations thread safe and remove synchronization in CacheLoaderInterceptor and CacheStoreInterceptor

Make cache loader implementations thread safe and remove synchronization in CacheLoaderInterceptor and CacheStoreInterceptor ---------------------------------------------------------------------------------------------------------------------------- Key: JBCACHE-1103 URL: http://jira.jboss.com/jira/browse/JBCACHE-1103 Project: JBoss Cache Issue Type: Bug Security Level: Public(Everyone can see) Components: Clustering, Cache loaders Affects Versions: 1.4.1.SP3 Reporter: Brian Stansberry Assigned To: Manik Surtani Fix For: 1.4.1.SP4 Can occur if 1) Thread1 on node1 is trying to get(fqnA) which is not in-VM, so the CCL tries to do a clustered get. The CacheLoaderInterceptor lock for fqnA is held at this time. 2) Thread1 blocks in FC waiting for credits. 3) Replication message for fqnA arrives from node2. 4) IncomingPacketHandler thread will block waiting for the CacheLoaderInterceptor lock for fqnA. 5) FC credits cannot arrive, so we deadlock. Stack traces below show this. Thread: TP-Processor623 : priority:5, demon:true, threadId:1637, threadState:TIMED_WAITING, threadLockName:EDU.oswego.cs.dl.util.concurrent.CondVar@5947cf38 java.lang.Object.wait(Native Method) EDU.oswego.cs.dl.util.concurrent.CondVar.timedwait(CondVar.java:222) org.jgroups.protocols.FC.handleDownMessage(FC.java:454) org.jgroups.protocols.FC.down(FC.java:374) org.jgroups.stack.Protocol.receiveDownEvent(Protocol.java:499) org.jgroups.protocols.FC.receiveDownEvent(FC.java:368) org.jgroups.stack.Protocol.passDown(Protocol.java:533) org.jgroups.protocols.FRAG2.down(FRAG2.java:167) org.jgroups.stack.Protocol.receiveDownEvent(Protocol.java:499) org.jgroups.stack.Protocol.passDown(Protocol.java:533) org.jgroups.protocols.pbcast.STATE_TRANSFER.down(STATE_TRANSFER.java:294) org.jgroups.stack.Protocol.receiveDownEvent(Protocol.java:499) org.jgroups.stack.ProtocolStack.down(ProtocolStack.java:390) org.jgroups.JChannel.down(JChannel.java:1231) org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.down(MessageDispatcher.java:790) org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.passDown(MessageDispatcher.java:767) org.jgroups.blocks.RequestCorrelator.sendRequest(RequestCorrelator.java:308) org.jgroups.blocks.GroupRequest.doExecute(GroupRequest.java:444) org.jgroups.blocks.GroupRequest.execute(GroupRequest.java:193) org.jgroups.blocks.MessageDispatcher.castMessage(MessageDispatcher.java:431) org.jgroups.blocks.RpcDispatcher.callRemoteMethods(RpcDispatcher.java:192) org.jgroups.blocks.RpcDispatcher.callRemoteMethods(RpcDispatcher.java:163) org.jboss.cache.TreeCache.callRemoteMethodsViaReflection(TreeCache.java:4437) org.jboss.cache.TreeCache.callRemoteMethods(TreeCache.java:4391) org.jboss.cache.loader.ClusteredCacheLoader.callRemote(ClusteredCacheLoader.java:75) org.jboss.cache.loader.ClusteredCacheLoader.get0(ClusteredCacheLoader.java:119) org.jboss.cache.loader.ClusteredCacheLoader.get(ClusteredCacheLoader.java:113) org.jboss.cache.interceptors.CacheLoaderInterceptor.loadData(CacheLoaderInterceptor.java:443) org.jboss.cache.interceptors.CacheLoaderInterceptor.loadNode(CacheLoaderInterceptor.java:353) org.jboss.cache.interceptors.CacheLoaderInterceptor.invoke(CacheLoaderInterceptor.java:180) org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) org.jboss.cache.interceptors.UnlockInterceptor.invoke(UnlockInterceptor.java:32) org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) org.jboss.cache.interceptors.ReplicationInterceptor.invoke(ReplicationInterceptor.java:39) org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) org.jboss.cache.interceptors.CacheStoreInterceptor.invoke(CacheStoreInterceptor.java:135) org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) org.jboss.cache.interceptors.TxInterceptor.handleNonTxMethod(TxInterceptor.java:365) org.jboss.cache.interceptors.TxInterceptor.invoke(TxInterceptor.java:160) org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) org.jboss.cache.interceptors.CacheMgmtInterceptor.invoke(CacheMgmtInterceptor.java:138) org.jboss.cache.TreeCache.invokeMethod(TreeCache.java:5863) org.jboss.cache.TreeCache.get(TreeCache.java:3627) org.jboss.cache.TreeCache.get(TreeCache.java:3608) org.jboss.cache.TreeCache.get(TreeCache.java:3404) sun.reflect.GeneratedMethodAccessor341.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) java.lang.reflect.Method.invoke(Method.java:597) org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155) org.jboss.mx.server.Invocation.dispatch(Invocation.java:94) org.jboss.mx.server.Invocation.invoke(Invocation.java:86) org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264) org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659) org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:210) $Proxy60.get(Unknown Source) org.jboss.web.tomcat.tc5.session.JBossCacheWrapper.get(JBossCacheWrapper.java:78) org.jboss.web.tomcat.tc5.session.JBossCacheService.loadSession(JBossCacheService.java:247) org.jboss.web.tomcat.tc5.session.JBossCacheManager.loadSession(JBossCacheManager.java:1006) org.jboss.web.tomcat.tc5.session.JBossCacheManager.findSession(JBossCacheManager.java:778) org.apache.catalina.connector.Request.doGetSession(Request.java:2196) org.apache.catalina.connector.Request.getSession(Request.java:2024) org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:97) org.jboss.web.tomcat.tc5.session.ClusteredSessionValve.invoke(ClusteredSessionValve.java:87) org.jboss.web.tomcat.tc5.session.JvmRouteValve.invoke(JvmRouteValve.java:84) org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:74) org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126) org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105) org.jboss.web.tomcat.tc5.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:156) org.jboss.web.tomcat.tc5.sso.ClusteredSingleSignOn.invoke(ClusteredSingleSignOn.java:637) org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107) org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148) org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:199) org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:282) org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:767) org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:697) org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java:889) org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:684) java.lang.Thread.run(Thread.java:619) Thread: IncomingPacketHandler (channel=Tomcat-LAB01) : priority:5, demon:true, threadId:35, threadState:WAITING, threadLockName:org.jboss.cache.interceptors.CacheLoaderInterceptor@5778e8ed java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:485) org.jboss.cache.interceptors.BaseCacheLoaderInterceptor.obtainLoaderLock(BaseCacheLoaderInterceptor.java:49) org.jboss.cache.interceptors.CacheLoaderInterceptor.invoke(CacheLoaderInterceptor.java:168) org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) org.jboss.cache.interceptors.UnlockInterceptor.invoke(UnlockInterceptor.java:32) org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) org.jboss.cache.interceptors.ReplicationInterceptor.invoke(ReplicationInterceptor.java:39) org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) org.jboss.cache.interceptors.CacheStoreInterceptor.invoke(CacheStoreInterceptor.java:64) org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) org.jboss.cache.interceptors.TxInterceptor.handleNonTxMethod(TxInterceptor.java:365) org.jboss.cache.interceptors.TxInterceptor.invoke(TxInterceptor.java:160) org.jboss.cache.interceptors.Interceptor.invoke(Interceptor.java:68) org.jboss.cache.interceptors.CacheMgmtInterceptor.invoke(CacheMgmtInterceptor.java:167) org.jboss.cache.TreeCache.invokeMethod(TreeCache.java:5863) org.jboss.cache.TreeCache._replicate(TreeCache.java:5162) sun.reflect.GeneratedMethodAccessor344.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) java.lang.reflect.Method.invoke(Method.java:597) org.jgroups.blocks.MethodCall.invoke(MethodCall.java:330) org.jgroups.blocks.RpcDispatcher.handle(RpcDispatcher.java:281) org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:654) org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:544) org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:367) org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:777) org.jgroups.JChannel.up(JChannel.java:1091) org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:382) org.jgroups.stack.ProtocolStack.receiveUpEvent(ProtocolStack.java:398) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:158) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.FRAG2.up(FRAG2.java:197) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.FC.up(FC.java:422) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.pbcast.GMS.up(GMS.java:768) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.protocols.pbcast.GMS.receiveUpEvent(GMS.java:788) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:262) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.UNICAST.up(UNICAST.java:259) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.pbcast.NAKACK.handleMessage(NAKACK.java:685) org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:517) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:170) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.FD.up(FD.java:300) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:301) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.MERGE2.up(MERGE2.java:162) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.Discovery.up(Discovery.java:225) org.jgroups.stack.Protocol.receiveUpEvent(Protocol.java:470) org.jgroups.stack.Protocol.passUp(Protocol.java:520) org.jgroups.protocols.TP.handleIncomingMessage(TP.java:909) org.jgroups.protocols.TP.handleIncomingPacket(TP.java:851) org.jgroups.protocols.TP.access$400(TP.java:46) org.jgroups.protocols.TP$IncomingPacketHandler.run(TP.java:1297) java.lang.Thread.run(Thread.java:619)