[infinispan-issues] [JBoss JIRA] (ISPN-5330) Various issues with stopping a cache/cachemanager

[ https://issues.jboss.org/browse/ISPN-5330?page=com.atlassian.jira.plugin.... ] William Burns updated ISPN-5330: -------------------------------- Description: While running stress test that constantly starts and stops a new cache manager I found several issues where a server would respond with incorrect values or exceptions. # The Marshaller objectTable can be nulled out causing a NPE. {code} Caused by: java.lang.NullPointerException at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:354) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.readParameters(ReplicableCommandExternalizer.java:101) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:84) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:1) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:434) at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:226) at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:354) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.readParameters(ReplicableCommandExternalizer.java:101) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:153) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:1) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:434) at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:226) at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:354) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.objectFromObjectStream(AbstractJBossMarshaller.java:135) at org.infinispan.marshall.core.VersionAwareMarshaller.objectFromByteBuffer(VersionAwareMarshaller.java:101) at org.infinispan.commons.marshall.AbstractDelegatingMarshaller.objectFromByteBuffer(AbstractDelegatingMarshaller.java:80) at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectFromBuffer(MarshallerAdapter.java:28) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:245) ... 22 more {code} # A command can be ran while a cache is STOPPING causing possibly invalid values to be returned. We need to most likely tweak the DistributionResponseGenerator for various commands to drop results that are null for GetKeyValueCommand for example (instead return UnsureResponse) # If a cache has gone to TERMINATING state an exception is thrown back to the caller which is majority of cases not handled {code} org.infinispan.remoting.RemoteException: ISPN000217: Received exception from main-NodeIJ-43170, see cause for remote stack trace at org.infinispan.remoting.transport.AbstractTransport.checkResponse(AbstractTransport.java:44) at org.infinispan.remoting.transport.AbstractTransport.parseResponseAndAddToResponseList(AbstractTransport.java:69) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:656) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:346) at org.infinispan.interceptors.distribution.BaseDistributionInterceptor.retrieveFromRemoteSources(BaseDistributionInterceptor.java:223) at org.infinispan.interceptors.distribution.NonTxDistributionInterceptor.visitGetAllCommand(NonTxDistributionInterceptor.java:142) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.interceptors.EntryWrappingInterceptor.visitGetAllCommand(EntryWrappingInterceptor.java:150) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitGetAllCommand(NonTransactionalLockingInterceptor.java:50) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111) at org.infinispan.commands.AbstractVisitor.visitGetAllCommand(AbstractVisitor.java:96) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.statetransfer.StateTransferInterceptor.visitGetAllCommand(StateTransferInterceptor.java:177) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.interceptors.CacheMgmtInterceptor.visitGetAllCommand(CacheMgmtInterceptor.java:125) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:102) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:71) at org.infinispan.commands.AbstractVisitor.visitGetAllCommand(AbstractVisitor.java:96) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) at org.infinispan.cache.impl.CacheImpl.getAll(CacheImpl.java:448) at org.infinispan.cache.impl.CacheImpl.getAll(CacheImpl.java:442) at org.infinispan.commands.GetAllCommandStressTest$1.call(GetAllCommandStressTest.java:117) at org.infinispan.commands.GetAllCommandStressTest$1.call(GetAllCommandStressTest.java:1) at org.infinispan.test.AbstractInfinispanTest$LoggingCallable.call(AbstractInfinispanTest.java:422) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: org.infinispan.IllegalLifecycleStateException: ISPN000324: Cache 'org.infinispan.commands.GetAllCommandStressTest' is in 'STOPPING' state and this is an invocation not belonging to an on-going transaction, so it does not accept new invocations. Either restart it or recreate the cache container. at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:91) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:71) at org.infinispan.commands.AbstractVisitor.visitGetAllCommand(AbstractVisitor.java:96) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) at org.infinispan.commands.remote.ClusteredGetManyCommand.perform(ClusteredGetManyCommand.java:83) at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokePerform(BasePerCacheInboundInvocationHandler.java:84) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:31) at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.handleRunnable(BasePerCacheInboundInvocationHandler.java:124) at org.infinispan.remoting.inboundhandler.NonTotalOrderPerCacheInboundInvocationHandler.handle(NonTotalOrderPerCacheInboundInvocationHandler.java:53) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleCacheRpcCommand(GlobalInboundInvocationHandler.java:125) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:74) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommandFromLocalCluster(CommandAwareRpcDispatcher.java:326) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:297) at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:460) at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:377) at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:250) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:675) at org.jgroups.JChannel.up(JChannel.java:739) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1029) at org.jgroups.protocols.RSVP.up(RSVP.java:201) at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) at org.jgroups.protocols.FlowControl.up(FlowControl.java:394) at org.jgroups.protocols.tom.TOA.up(TOA.java:121) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1042) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1064) at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:779) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:652) at org.jgroups.protocols.Discovery.up(Discovery.java:291) at org.jgroups.protocols.TP.passMessageUp(TP.java:1577) at org.jgroups.protocols.TP$MyHandler.run(TP.java:1796) ... 3 more {code} was: While running stress test that constantly starts and stops a new cache manager I found several issues where a server would respond with incorrect values or exceptions. # The Marshaller objectTable can be nulled out causing a NPE. I don't seem to have the stack trace, hopefully will update later. # A command can be ran while a cache is STOPPING causing possibly invalid values to be returned. We need to most likely tweak the DistributionResponseGenerator for various commands to drop results that are null for GetKeyValueCommand for example (instead return UnsureResponse) # If a cache has gone to TERMINATING state an exception is thrown back to the caller which is majority of cases not handled {code} org.infinispan.remoting.RemoteException: ISPN000217: Received exception from main-NodeIJ-43170, see cause for remote stack trace at org.infinispan.remoting.transport.AbstractTransport.checkResponse(AbstractTransport.java:44) at org.infinispan.remoting.transport.AbstractTransport.parseResponseAndAddToResponseList(AbstractTransport.java:69) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:656) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:346) at org.infinispan.interceptors.distribution.BaseDistributionInterceptor.retrieveFromRemoteSources(BaseDistributionInterceptor.java:223) at org.infinispan.interceptors.distribution.NonTxDistributionInterceptor.visitGetAllCommand(NonTxDistributionInterceptor.java:142) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.interceptors.EntryWrappingInterceptor.visitGetAllCommand(EntryWrappingInterceptor.java:150) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitGetAllCommand(NonTransactionalLockingInterceptor.java:50) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111) at org.infinispan.commands.AbstractVisitor.visitGetAllCommand(AbstractVisitor.java:96) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.statetransfer.StateTransferInterceptor.visitGetAllCommand(StateTransferInterceptor.java:177) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.interceptors.CacheMgmtInterceptor.visitGetAllCommand(CacheMgmtInterceptor.java:125) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:102) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:71) at org.infinispan.commands.AbstractVisitor.visitGetAllCommand(AbstractVisitor.java:96) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) at org.infinispan.cache.impl.CacheImpl.getAll(CacheImpl.java:448) at org.infinispan.cache.impl.CacheImpl.getAll(CacheImpl.java:442) at org.infinispan.commands.GetAllCommandStressTest$1.call(GetAllCommandStressTest.java:117) at org.infinispan.commands.GetAllCommandStressTest$1.call(GetAllCommandStressTest.java:1) at org.infinispan.test.AbstractInfinispanTest$LoggingCallable.call(AbstractInfinispanTest.java:422) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: org.infinispan.IllegalLifecycleStateException: ISPN000324: Cache 'org.infinispan.commands.GetAllCommandStressTest' is in 'STOPPING' state and this is an invocation not belonging to an on-going transaction, so it does not accept new invocations. Either restart it or recreate the cache container. at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:91) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:71) at org.infinispan.commands.AbstractVisitor.visitGetAllCommand(AbstractVisitor.java:96) at org.infinispan.commands.read.GetAllCommand.acceptVisitor(GetAllCommand.java:59) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) at org.infinispan.commands.remote.ClusteredGetManyCommand.perform(ClusteredGetManyCommand.java:83) at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.invokePerform(BasePerCacheInboundInvocationHandler.java:84) at org.infinispan.remoting.inboundhandler.BaseBlockingRunnable.run(BaseBlockingRunnable.java:31) at org.infinispan.remoting.inboundhandler.BasePerCacheInboundInvocationHandler.handleRunnable(BasePerCacheInboundInvocationHandler.java:124) at org.infinispan.remoting.inboundhandler.NonTotalOrderPerCacheInboundInvocationHandler.handle(NonTotalOrderPerCacheInboundInvocationHandler.java:53) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleCacheRpcCommand(GlobalInboundInvocationHandler.java:125) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:74) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommandFromLocalCluster(CommandAwareRpcDispatcher.java:326) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:297) at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:460) at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:377) at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:250) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:675) at org.jgroups.JChannel.up(JChannel.java:739) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1029) at org.jgroups.protocols.RSVP.up(RSVP.java:201) at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) at org.jgroups.protocols.FlowControl.up(FlowControl.java:394) at org.jgroups.protocols.tom.TOA.up(TOA.java:121) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1042) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1064) at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:779) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:652) at org.jgroups.protocols.Discovery.up(Discovery.java:291) at org.jgroups.protocols.TP.passMessageUp(TP.java:1577) at org.jgroups.protocols.TP$MyHandler.run(TP.java:1796) ... 3 more {code} -- This message was sent by Atlassian JIRA (v6.3.11#6341)