CLONE - "IllegalStateException: repl: Received prepare cache view request, but we are not a member" --------------------------------------------------------------------------------------------------- Key: AS7-5381 URL: https://issues.jboss.org/browse/AS7-5381 Project: Application Server 7 Issue Type: Bug Components: Clustering Environment: affected 6.0.1 DR1 CI build Reporter: Radoslav Husar Assignee: Paul Ferraro Priority: Blocker Labels: as713tracking, eap601candidate Fix For: 7.1.3.Final (EAP) We are seeing problems with view installs. Following this event, all (or most) subsequent requests to that node fail with response code 500. {noformat} [JBossINF] 13:26:57,797 INFO [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (Incoming-9,null) ISPN000094: Received new cluster view: [perf21/web|8] [perf21/web, perf18/web, perf19/web] [JBossINF] 13:26:57,838 WARN [org.infinispan.commands.control.CacheViewControlCommand] (OOB-163,null) ISPN000071: Caught exception when handling command CacheViewControlCommand{cache=default-host/clusterbench, type=PREPARE_VIEW, sender=perf20/web, newViewId=13, newMembers=[perf21/web, perf18/web, perf19/web], oldViewId=12, oldMembers=[perf20/web, perf21/web, perf18/web, perf19/web]}: org.infinispan.statetransfer.StateTransferCancelledException [JBossINF] at org.infinispan.statetransfer.BaseStateTransferTask.checkIfCancelled(BaseStateTransferTask.java:151) [JBossINF] at org.infinispan.statetransfer.DistributedStateTransferTask.doPerformStateTransfer(DistributedStateTransferTask.java:124) [JBossINF] at org.infinispan.statetransfer.BaseStateTransferTask.performStateTransfer(BaseStateTransferTask.java:93) [JBossINF] at org.infinispan.statetransfer.BaseStateTransferManagerImpl.prepareView(BaseStateTransferManagerImpl.java:331) [JBossINF] at org.infinispan.cacheviews.CacheViewsManagerImpl.handlePrepareView(CacheViewsManagerImpl.java:486) [JBossINF] at org.infinispan.commands.control.CacheViewControlCommand.perform(CacheViewControlCommand.java:126) [JBossINF] at org.infinispan.remoting.InboundInvocationHandlerImpl.handle(InboundInvocationHandlerImpl.java:95) [JBossINF] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommand(CommandAwareRpcDispatcher.java:226) [JBossINF] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:203) [JBossINF] at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:456) [JBossINF] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:363) [JBossINF] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:238) [JBossINF] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:601) [JBossINF] at org.jgroups.blocks.mux.MuxUpHandler.up(MuxUpHandler.java:130) [JBossINF] at org.jgroups.JChannel.up(JChannel.java:716) [JBossINF] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1026) [JBossINF] at org.jgroups.protocols.RSVP.up(RSVP.java:179) [JBossINF] at org.jgroups.protocols.FRAG2.up(FRAG2.java:181) [JBossINF] at org.jgroups.protocols.FlowControl.up(FlowControl.java:418) [JBossINF] at org.jgroups.protocols.FlowControl.up(FlowControl.java:400) [JBossINF] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:889) [JBossINF] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:244) [JBossINF] at org.jgroups.protocols.UNICAST2.handleDataReceived(UNICAST2.java:759) [JBossINF] at org.jgroups.protocols.UNICAST2.up(UNICAST2.java:365) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:602) [JBossINF] at org.jgroups.protocols.BARRIER.up(BARRIER.java:102) [JBossINF] at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:143) [JBossINF] at org.jgroups.protocols.FD.up(FD.java:273) [JBossINF] at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:288) [JBossINF] at org.jgroups.protocols.MERGE2.up(MERGE2.java:205) [JBossINF] at org.jgroups.protocols.Discovery.up(Discovery.java:359) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:363) [JBossINF] at org.jgroups.protocols.TP.passMessageUp(TP.java:1185) [JBossINF] at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1733) [JBossINF] at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1715) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [rt.jar:1.6.0_30] [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [rt.jar:1.6.0_30] [JBossINF] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_30] {noformat} and {noformat} [JBossINF] 13:27:48,163 WARN [org.infinispan.commands.control.CacheViewControlCommand] (OOB-14,null) ISPN000071: Caught exception when handling command CacheViewControlCommand{cache=remote-connector-client-mappings, type=PREPARE_VIEW, sender=perf21/ejb, newViewId=19, newMembers=[perf18/ejb, perf21/ejb, perf19/ejb], oldViewId=18, oldMembers=[perf18/ejb, perf21/ejb, perf19/ejb]}: java.lang.IllegalStateException: remote-connector-client-mappings: Received prepare cache view request, but we are not a member. View is CacheView{viewId=19, members=[perf18/ejb, perf21/ejb, perf19/ejb]} [JBossINF] at org.infinispan.cacheviews.CacheViewsManagerImpl.handlePrepareView(CacheViewsManagerImpl.java:473) [JBossINF] at org.infinispan.commands.control.CacheViewControlCommand.perform(CacheViewControlCommand.java:126) [JBossINF] at org.infinispan.remoting.InboundInvocationHandlerImpl.handle(InboundInvocationHandlerImpl.java:95) [JBossINF] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.executeCommand(CommandAwareRpcDispatcher.java:226) [JBossINF] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:203) [JBossINF] at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:456) [JBossINF] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:363) [JBossINF] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:238) [JBossINF] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:601) [JBossINF] at org.jgroups.blocks.mux.MuxUpHandler.up(MuxUpHandler.java:130) [JBossINF] at org.jgroups.JChannel.up(JChannel.java:716) [JBossINF] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1026) [JBossINF] at org.jgroups.protocols.RSVP.up(RSVP.java:179) [JBossINF] at org.jgroups.protocols.FRAG2.up(FRAG2.java:181) [JBossINF] at org.jgroups.protocols.FlowControl.up(FlowControl.java:400) [JBossINF] at org.jgroups.protocols.FlowControl.up(FlowControl.java:418) [JBossINF] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:889) [JBossINF] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:244) [JBossINF] at org.jgroups.protocols.UNICAST2.up(UNICAST2.java:383) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK.handleMessage(NAKACK.java:706) [JBossINF] at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:566) [JBossINF] at org.jgroups.protocols.BARRIER.up(BARRIER.java:126) [JBossINF] at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:143) [JBossINF] at org.jgroups.protocols.FD.up(FD.java:273) [JBossINF] at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:288) [JBossINF] at org.jgroups.protocols.MERGE2.up(MERGE2.java:205) [JBossINF] at org.jgroups.protocols.Discovery.up(Discovery.java:359) [JBossINF] at org.jgroups.stack.Protocol.up(Protocol.java:363) [JBossINF] at org.jgroups.protocols.TP.passMessageUp(TP.java:1185) [JBossINF] at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1733) [JBossINF] at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1715) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [rt.jar:1.6.0_30] [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [rt.jar:1.6.0_30] [JBossINF] at java.lang.Thread.run(Thread.java:662) [rt.jar:1.6.0_30] {noformat} Seen here https://jenkins.mw.lab.eng.bos.redhat.com/hudson/view/EAP6/view/EAP6-Fail...