[jboss-jira] [JBoss JIRA] (WFLY-5144) OutdatedTopologyException: Node has a newer topology id

OutdatedTopologyException: Node has a newer topology id ------------------------------------------------------- Key: WFLY-5144 URL: https://issues.jboss.org/browse/WFLY-5144 Project: WildFly Issue Type: Bug Components: Clustering Affects Versions: 10.0.0.Beta1 Reporter: Michal Vinkler Assignee: Paul Ferraro Seen regularly in our failover tests, with *distributed* cache used. After failing one node (perf18), perf19 gets a topology update and immediately logs this error: {code} [JBossINF] [0m[0m02:58:39,000 INFO [org.infinispan.CLUSTER] (transport-thread--p8-t12) ISPN000310: Starting cluster-wide rebalance for cache clusterbench-ee7.ear.clusterbench-ee7-web-default.war, topology CacheTopology{id=10, rebalanceId=5, currentCH=DefaultConsistentHash{ns=80, owners = (3)[perf19: 30+10, perf20: 24+16, perf21: 26+14]}, pendingCH=DefaultConsistentHash{ns=80, owners = (3)[perf19: 27+26, perf20: 26+28, perf21: 27+26]}, unionCH=null, actualMembers=[perf19, perf20, perf21]} [JBossINF] [0m[0m02:58:39,002 INFO [org.infinispan.CLUSTER] (transport-thread--p8-t13) ISPN000310: Starting cluster-wide rebalance for cache dist, topology CacheTopology{id=10, rebalanceId=5, currentCH=DefaultConsistentHash{ns=80, owners = (3)[perf19: 30+10, perf20: 24+16, perf21: 26+14]}, pendingCH=DefaultConsistentHash{ns=80, owners = (3)[perf19: 27+26, perf20: 26+28, perf21: 27+26]}, unionCH=null, actualMembers=[perf19, perf20, perf21]} [JBossINF] [0m[31m02:58:40,293 ERROR [io.undertow.request] (default task-103) UT005023: Exception handling request to /clusterbench/session: org.infinispan.statetransfer.OutdatedTopologyException: Node perf21 has a newer topology id [JBossINF] at org.infinispan.interceptors.distribution.TxDistributionInterceptor.checkTxCommandResponses(TxDistributionInterceptor.java:263) [JBossINF] at org.infinispan.interceptors.distribution.TxDistributionInterceptor.visitLockControlCommand(TxDistributionInterceptor.java:182) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitLockControlCommand(AbstractVisitor.java:174) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitLockControlCommand(AbstractVisitor.java:174) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitLockControlCommand(AbstractVisitor.java:174) [JBossINF] at org.infinispan.commands.control.LockControlCommand.acceptVisitor(LockControlCommand.java:110) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) [JBossINF] at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.acquireRemoteIfNeeded(PessimisticLockingInterceptor.java:238) [JBossINF] at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitDataReadCommand(PessimisticLockingInterceptor.java:66) [JBossINF] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitGetKeyValueCommand(AbstractLockingInterceptor.java:70) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:86) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) [JBossINF] at org.infinispan.interceptors.TxInterceptor.enlistReadAndInvokeNext(TxInterceptor.java:287) [JBossINF] at org.infinispan.interceptors.TxInterceptor.visitGetKeyValueCommand(TxInterceptor.java:259) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:86) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:360) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleDefault(StateTransferInterceptor.java:345) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:86) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) [JBossINF] at org.infinispan.interceptors.CacheMgmtInterceptor.visitDataReadCommand(CacheMgmtInterceptor.java:103) [JBossINF] at org.infinispan.interceptors.CacheMgmtInterceptor.visitGetKeyValueCommand(CacheMgmtInterceptor.java:91) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:102) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:71) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:86) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) [JBossINF] at org.infinispan.cache.impl.CacheImpl.get(CacheImpl.java:430) [JBossINF] at org.infinispan.cache.impl.DecoratedCache.get(DecoratedCache.java:427) [JBossINF] at org.infinispan.cache.impl.AbstractDelegatingCache.get(AbstractDelegatingCache.java:287) [JBossINF] at org.wildfly.clustering.web.infinispan.session.coarse.CoarseSessionFactory.findValue(CoarseSessionFactory.java:120) [JBossINF] at org.wildfly.clustering.web.infinispan.session.coarse.CoarseSessionFactory.findValue(CoarseSessionFactory.java:56) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionManager.findSession(InfinispanSessionManager.java:238) [JBossINF] at org.wildfly.clustering.web.undertow.session.DistributableSessionManager.getSession(DistributableSessionManager.java:124) [JBossINF] at io.undertow.servlet.spec.ServletContextImpl.getSession(ServletContextImpl.java:725) [JBossINF] at io.undertow.servlet.spec.HttpServletRequestImpl.getSession(HttpServletRequestImpl.java:367) [JBossINF] at org.jboss.weld.servlet.SessionHolder.requestInitialized(SessionHolder.java:47) [JBossINF] at org.jboss.weld.servlet.HttpContextLifecycle.requestInitialized(HttpContextLifecycle.java:231) [JBossINF] at org.jboss.weld.servlet.WeldInitialListener.requestInitialized(WeldInitialListener.java:152) [JBossINF] at io.undertow.servlet.core.ApplicationListeners.requestInitialized(ApplicationListeners.java:216) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:281) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:261) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:80) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:172) [JBossINF] at io.undertow.server.Connectors.executeRootHandler(Connectors.java:199) [JBossINF] at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:778) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) {code} perf20,21 log the same error, although stating that it is *perf19* that has a newer topology id. Server logs: http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-7x-failover-http-... http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-7x-failover-http-... http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-7x-failover-http-...