NPE caused by missing Externalizer Table ---------------------------------------- Key: ISPN-4701 URL: https://issues.jboss.org/browse/ISPN-4701 Project: Infinispan Issue Type: Bug Components: Core Affects Versions: 7.0.0.Beta1 Reporter: Sanne Grinovero Fix For: 8.0.0.Beta1 I guess this is a race condition as I can only trigger it during my StressTest which dynamically kills and starts new nodes: {noformat}2014-09-09 11:11:22,133 WARN [org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher] (OOB-7,IndexManagerStressTest-NodeO-45921) ISPN000220: Problems un-marshalling remote command from byte buffer java.lang.NullPointerException at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.marshall.exts.ListExternalizer.readObject(ListExternalizer.java:62) at org.infinispan.marshall.exts.ListExternalizer.readObject(ListExternalizer.java:26) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:409) at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:214) at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.statetransfer.StateChunk$Externalizer.readObject(StateChunk.java:88) at org.infinispan.statetransfer.StateChunk$Externalizer.readObject(StateChunk.java:65) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:409) at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:214) at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.marshall.exts.ListExternalizer.readObject(ListExternalizer.java:62) at org.infinispan.marshall.exts.ListExternalizer.readObject(ListExternalizer.java:26) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:409) at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:214) at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.readParameters(ReplicableCommandExternalizer.java:101) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:155) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:65) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:409) at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:214) at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.objectFromObjectStream(AbstractJBossMarshaller.java:135) at org.infinispan.marshall.core.VersionAwareMarshaller.objectFromByteBuffer(VersionAwareMarshaller.java:101) at org.infinispan.commons.marshall.AbstractDelegatingMarshaller.objectFromByteBuffer(AbstractDelegatingMarshaller.java:80) at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectFromBuffer(MarshallerAdapter.java:28) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:206) at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:460) at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:377) at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:250) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:674) at org.jgroups.JChannel.up(JChannel.java:733) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1030) at org.jgroups.protocols.RSVP.up(RSVP.java:190) at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) at org.jgroups.protocols.FlowControl.up(FlowControl.java:390) at org.jgroups.protocols.tom.TOA.up(TOA.java:121) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1042) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1034) at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:752) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:399) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:610) at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:297) at org.jgroups.protocols.Discovery.up(Discovery.java:277) at org.jgroups.protocols.TP.passMessageUp(TP.java:1559) at org.jgroups.protocols.TP$MyHandler.run(TP.java:1778) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2014-09-09 11:11:22,138 ERROR [org.infinispan.statetransfer.OutboundTransferTask] (transport-thread-IndexManagerStressTest-NodeO-p86-t3) Failed to send entries to node IndexManagerStressTest-NodeN-61517 : java.lang.RuntimeException: Failure to marshal argument(s) org.infinispan.commons.CacheException: java.lang.RuntimeException: Failure to marshal argument(s) at org.infinispan.commons.util.Util.rewrapAsCacheException(Util.java:655) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:176) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:536) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:281) at org.infinispan.statetransfer.OutboundTransferTask.sendEntries(OutboundTransferTask.java:239) at org.infinispan.statetransfer.OutboundTransferTask.sendEntry(OutboundTransferTask.java:195) at org.infinispan.statetransfer.OutboundTransferTask.run(OutboundTransferTask.java:149) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Unknown Source) Caused by: java.lang.RuntimeException: Failure to marshal argument(s) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:354) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:369) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:167) ... 12 more Caused by: java.lang.NullPointerException at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.getObjectWriter(JBossMarshaller.java:143) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:138) at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) at org.infinispan.container.entries.ImmortalCacheEntry$Externalizer.writeObject(ImmortalCacheEntry.java:146) at org.infinispan.container.entries.ImmortalCacheEntry$Externalizer.writeObject(ImmortalCacheEntry.java:142) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:415) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:144) at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) at org.infinispan.commons.marshall.MarshallUtil.marshallCollection(MarshallUtil.java:25) at org.infinispan.marshall.exts.ListExternalizer.writeObject(ListExternalizer.java:44) at org.infinispan.marshall.exts.ListExternalizer.writeObject(ListExternalizer.java:26) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:415) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:144) at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) at org.infinispan.statetransfer.StateChunk$Externalizer.writeObject(StateChunk.java:80) at org.infinispan.statetransfer.StateChunk$Externalizer.writeObject(StateChunk.java:65) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:415) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:144) at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) at org.infinispan.commons.marshall.MarshallUtil.marshallCollection(MarshallUtil.java:25) at org.infinispan.marshall.exts.ListExternalizer.writeObject(ListExternalizer.java:44) at org.infinispan.marshall.exts.ListExternalizer.writeObject(ListExternalizer.java:26) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:415) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:144) at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.writeCommandParameters(ReplicableCommandExternalizer.java:58) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.marshallParameters(CacheRpcCommandExternalizer.java:125) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.writeObject(CacheRpcCommandExternalizer.java:109) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.writeObject(CacheRpcCommandExternalizer.java:65) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:415) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:144) at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.objectToObjectStream(AbstractJBossMarshaller.java:73) at org.infinispan.marshall.core.VersionAwareMarshaller.objectToBuffer(VersionAwareMarshaller.java:77) at org.infinispan.commons.marshall.AbstractMarshaller.objectToBuffer(AbstractMarshaller.java:41) at org.infinispan.commons.marshall.AbstractDelegatingMarshaller.objectToBuffer(AbstractDelegatingMarshaller.java:85) at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectToBuffer(MarshallerAdapter.java:23) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:352) ... 14 more Caused by: an exception which occurred: in object org.infinispan.query.distributed.Person@5544c0ae in object org.infinispan.container.entries.ImmortalCacheEntry@24992efc in object java.util.ArrayList@82fa0d17 in object org.infinispan.statetransfer.StateChunk@1d7732e9 in object java.util.ArrayList@d08f1eef in object org.infinispan.statetransfer.StateResponseCommand@3a446ef4{noformat}