[infinispan-issues] [JBoss JIRA] (ISPN-6325) Other node shutting down caused local exception

Other node shutting down caused local exception ----------------------------------------------- Key: ISPN-6325 URL: https://issues.redhat.com/browse/ISPN-6325 Project: Infinispan Issue Type: Bug Affects Versions: 8.1.2.Final Reporter: Will Burns Priority: Major I was running DistributedStreamRehashStressTest and I found that one of my processing threads was killed by the following exception, which shouldn't be propagated to the user: {code} java.lang.AssertionError: Found an exception in at least 1 thread at org.testng.Assert.fail(Assert.java:83) at org.infinispan.stream.stress.DistributedStreamRehashStressTest.testStressNodesLeavingWhilePerformingCallable(DistributedStreamRehashStressTest.java:226) at org.infinispan.stream.stress.DistributedStreamRehashStressTest.testStressNodesLeavingWhileMultipleIterators(DistributedStreamRehashStressTest.java:113) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84) at org.testng.internal.Invoker.invokeMethod(Invoker.java:714) at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901) at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231) at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127) at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111) at org.testng.TestRunner.privateRun(TestRunner.java:767) at org.testng.TestRunner.run(TestRunner.java:617) at org.testng.SuiteRunner.runTest(SuiteRunner.java:348) at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:343) at org.testng.SuiteRunner.privateRun(SuiteRunner.java:305) at org.testng.SuiteRunner.run(SuiteRunner.java:254) at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:52) at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:86) at org.testng.TestNG.runSuitesSequentially(TestNG.java:1224) at org.testng.TestNG.runSuitesLocally(TestNG.java:1149) at org.testng.TestNG.run(TestNG.java:1057) at org.testng.IDEARemoteTestNG.run(IDEARemoteTestNG.java:72) at org.testng.RemoteTestNGStarter.main(RemoteTestNGStarter.java:122) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) Caused by: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from DistributedStreamRehashStressTest-NodeCU-43302, see cause for remote stack trace at org.infinispan.remoting.transport.AbstractTransport.checkResponse(AbstractTransport.java:44) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:796) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$212(JGroupsTransport.java:633) at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602) at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) at org.infinispan.remoting.transport.jgroups.SingleResponseFuture.futureDone(SingleResponseFuture.java:30) at org.jgroups.blocks.Request.checkCompletion(Request.java:162) at org.jgroups.blocks.UnicastRequest.receiveResponse(UnicastRequest.java:81) at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:373) at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:237) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:695) at org.jgroups.JChannel.up(JChannel.java:738) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1030) at org.jgroups.protocols.RSVP.up(RSVP.java:201) at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) at org.jgroups.protocols.FlowControl.up(FlowControl.java:392) at org.jgroups.protocols.tom.TOA.up(TOA.java:121) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1043) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1064) at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:779) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:652) at org.jgroups.protocols.Discovery.up(Discovery.java:296) at org.jgroups.protocols.TP.passMessageUp(TP.java:1590) at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1802) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from DistributedStreamRehashStressTest-NodeCV-40406, see cause for remote stack trace ... 31 more Caused by: org.infinispan.commons.CacheException: Problems invoking command. at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:180) at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:402) at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:352) ... 20 more Caused by: java.io.EOFException: Read past end of file at org.jboss.marshalling.SimpleDataInput.eofOnRead(SimpleDataInput.java:151) at org.jboss.marshalling.SimpleDataInput.readUnsignedByteDirect(SimpleDataInput.java:294) at org.jboss.marshalling.SimpleDataInput.readUnsignedByte(SimpleDataInput.java:249) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.commons.marshall.MarshallUtil.unmarshallCollectionUnbounded(MarshallUtil.java:217) at org.infinispan.stream.impl.StreamSegmentResponseCommand.readFrom(StreamSegmentResponseCommand.java:61) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:113) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:173) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:68) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:478) at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:235) at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:149) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:354) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41) at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.objectFromObjectStream(AbstractJBossMarshaller.java:134) at org.infinispan.marshall.core.VersionAwareMarshaller.objectFromByteBuffer(VersionAwareMarshaller.java:101) at org.infinispan.commons.marshall.AbstractDelegatingMarshaller.objectFromByteBuffer(AbstractDelegatingMarshaller.java:80) at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectFromBuffer(MarshallerAdapter.java:28) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:160) ... 22 more {code}