[infinispan-issues] [JBoss JIRA] (ISPN-7184) Server startup can fail after the upgrade to JGroups 4.0.0

Server startup can fail after the upgrade to JGroups 4.0.0 ---------------------------------------------------------- Key: ISPN-7184 URL: https://issues.jboss.org/browse/ISPN-7184 Project: Infinispan Issue Type: Bug Components: Server Reporter: Jakub Markos Assignee: Dan Berindei Fix For: 9.0.0.Beta1 During cluster startup, one of the nodes throws this ERROR: {code} 04:34:27,390 INFO [org.infinispan.factories.GlobalComponentRegistry] (MSC service thread 1-6) ISPN000128: Infinispan version: Infinispan 'Ruppaner' 9.0.0-SNAPSHOT 04:34:27,509 ERROR [org.jgroups.blocks.RequestCorrelator] (jgroups-14,slave0) JGRP000225: failed unmarshalling buffer into return value: java.io.EOFException: Read past end of file at org.jboss.marshalling.SimpleDataInput.eofOnRead(SimpleDataInput.java:151) at org.jboss.marshalling.SimpleDataInput.readUnsignedByteDirect(SimpleDataInput.java:294) at org.jboss.marshalling.SimpleDataInput.readUnsignedByte(SimpleDataInput.java:249) at org.jboss.marshalling.river.RiverUnmarshaller.start(RiverUnmarshaller.java:1247) at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.startObjectInput(AbstractJBossMarshaller.java:128) at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.objectFromByteBuffer(AbstractJBossMarshaller.java:109) at org.infinispan.marshall.core.GlobalMarshaller.objectFromByteBuffer(GlobalMarshaller.java:88) at org.infinispan.remoting.transport.jgroups.CustomRequestCorrelator.handleResponse(CustomRequestCorrelator.java:38) at org.jgroups.blocks.RequestCorrelator.dispatch(RequestCorrelator.java:366) at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:310) at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:579) at org.jgroups.JChannel.up(JChannel.java:792) at org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:134) at org.jgroups.stack.Protocol.up(Protocol.java:383) at org.jgroups.protocols.FORK.up(FORK.java:134) at org.jgroups.protocols.FRAG2.up(FRAG2.java:159) at org.jgroups.protocols.FlowControl.up(FlowControl.java:359) at org.jgroups.protocols.FlowControl.up(FlowControl.java:367) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:872) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:240) at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1053) at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:771) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:423) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:599) at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:129) at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:196) at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:252) at org.jgroups.protocols.MERGE3.up(MERGE3.java:272) at org.jgroups.protocols.Discovery.up(Discovery.java:270) at org.jgroups.protocols.TP.passMessageUp(TP.java:1169) at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1385) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) {code} Nasty stuff happens afterwards on other nodes, like: {code} 04:35:28,434 ERROR [org.jboss.msc.service.fail] (MSC service thread 1-8) MSC000001: Failed to start service jboss.datagrid-infinispan.clustered.rest: org.jboss.msc.service.StartException in service jboss.datagrid-infinispan.clustered.rest: Failed to start service at org.jboss.msc.service.ServiceControllerImpl$StartTask.run(ServiceControllerImpl.java:1904) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.infinispan.commons.CacheException: Unable to invoke method public void org.infinispan.statetransfer.StateTransferManagerImpl.start() throws java.lang.Exception on object of type StateTransferManagerImpl at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:172) at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:867) at org.infinispan.factories.AbstractComponentRegistry.invokeStartMethods(AbstractComponentRegistry.java:633) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:622) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:547) at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:231) at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:808) at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:639) at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:590) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:454) at org.infinispan.manager.impl.AbstractDelegatingEmbeddedCacheManager.getCache(AbstractDelegatingEmbeddedCacheManager.java:133) at org.infinispan.server.infinispan.SecurityActions$5.run(SecurityActions.java:131) at org.infinispan.server.infinispan.SecurityActions$5.run(SecurityActions.java:128) at org.infinispan.security.Security.doPrivileged(Security.java:76) at org.infinispan.server.infinispan.SecurityActions.doPrivileged(SecurityActions.java:64) at org.infinispan.server.infinispan.SecurityActions.startCache(SecurityActions.java:136) at org.jboss.as.clustering.infinispan.subsystem.CacheService.start(CacheService.java:86) at org.jboss.msc.service.ServiceControllerImpl$StartTask.startService(ServiceControllerImpl.java:1948) at org.jboss.msc.service.ServiceControllerImpl$StartTask.run(ServiceControllerImpl.java:1881) ... 3 more Caused by: org.infinispan.util.concurrent.TimeoutException: Replication timeout for slave0 at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:838) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:635) at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602) at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) at org.infinispan.remoting.transport.jgroups.SingleResponseFuture.call(SingleResponseFuture.java:50) at org.infinispan.remoting.transport.jgroups.SingleResponseFuture.call(SingleResponseFuture.java:15) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ... 3 more {code}