[jboss-jira] [JBoss JIRA] (WFLY-6087) AbstractCacheTransaction is prone to race conditions, NPE in containsLockOrBackupLock(Object)

AbstractCacheTransaction is prone to race conditions, NPE in containsLockOrBackupLock(Object) --------------------------------------------------------------------------------------------- Key: WFLY-6087 URL: https://issues.jboss.org/browse/WFLY-6087 Project: WildFly Issue Type: Bug Components: Clustering Affects Versions: 10.0.0.CR4 Reporter: Radoslav Husar Assignee: Radoslav Husar Seen in failover-ejb-ejbservlet-jvmkill-dist-sync scenario. 6 seconds after failing perf20 (jvmkill), perf18 logged ISPN000136: Execution error: java.lang.NullPointerException. Timeline: {code:title=client} 2015/11/27 08:23:25:206 EST [INFO ][TestController] HOST perf17.mw.lab.eng.bos.redhat.com:rootProcess:c - Node 2 (perf20) is down. {code} {code:title=perf18} [JBossINF] [0m[31m08:23:31,049 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (default task-70) ISPN000136: Execution error: java.lang.NullPointerException [JBossINF] at org.infinispan.transaction.impl.AbstractCacheTransaction.containsLockOrBackupLock(AbstractCacheTransaction.java:254) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.lambda$getTransactionWithLockedKey$177(DefaultPendingLockManager.java:252) [JBossINF] at org.infinispan.commons.util.concurrent.jdk8backported.EquivalentConcurrentHashMapV8$ValuesView.forEach(EquivalentConcurrentHashMapV8.java:4604) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.forEachTransaction(DefaultPendingLockManager.java:284) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.getTransactionWithLockedKey(DefaultPendingLockManager.java:249) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.checkForPendingLock(DefaultPendingLockManager.java:187) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.awaitPendingTransactionsForKey(DefaultPendingLockManager.java:115) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.checkPendingAndLockKey(AbstractTxLockingInterceptor.java:190) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.lockOrRegisterBackupLock(AbstractTxLockingInterceptor.java:115) [JBossINF] at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitDataReadCommand(PessimisticLockingInterceptor.java:69) [JBossINF] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitGetKeyValueCommand(AbstractLockingInterceptor.java:76) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:85) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.TxInterceptor.enlistReadAndInvokeNext(TxInterceptor.java:345) [JBossINF] at org.infinispan.interceptors.TxInterceptor.visitGetKeyValueCommand(TxInterceptor.java:330) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:85) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:403) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleDefault(StateTransferInterceptor.java:388) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:85) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:102) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:71) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:85) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:85) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) [JBossINF] at org.infinispan.cache.impl.CacheImpl.get(CacheImpl.java:412) [JBossINF] at org.infinispan.cache.impl.DecoratedCache.get(DecoratedCache.java:437) [JBossINF] at org.infinispan.cache.impl.AbstractDelegatingCache.get(AbstractDelegatingCache.java:286) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionMetaDataFactory.getValue(InfinispanSessionMetaDataFactory.java:70) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionMetaDataFactory.findValue(InfinispanSessionMetaDataFactory.java:60) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionMetaDataFactory.findValue(InfinispanSessionMetaDataFactory.java:36) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionFactory.findValue(InfinispanSessionFactory.java:61) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionFactory.findValue(InfinispanSessionFactory.java:40) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionManager.findSession(InfinispanSessionManager.java:234) [JBossINF] at org.wildfly.clustering.web.undertow.session.DistributableSessionManager.getSession(DistributableSessionManager.java:140) [JBossINF] at io.undertow.servlet.spec.ServletContextImpl.getSession(ServletContextImpl.java:725) [JBossINF] at io.undertow.servlet.spec.HttpServletRequestImpl.getSession(HttpServletRequestImpl.java:370) [JBossINF] at org.jboss.weld.servlet.SessionHolder.requestInitialized(SessionHolder.java:47) [JBossINF] at org.jboss.weld.servlet.HttpContextLifecycle.requestInitialized(HttpContextLifecycle.java:231) [JBossINF] at org.jboss.weld.servlet.WeldInitialListener.requestInitialized(WeldInitialListener.java:152) [JBossINF] at io.undertow.servlet.core.ApplicationListeners.requestInitialized(ApplicationListeners.java:245) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:283) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:263) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:81) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:174) [JBossINF] at io.undertow.server.Connectors.executeRootHandler(Connectors.java:202) [JBossINF] at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:793) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) {code} Another NPE followed right after: {code} [JBossINF] [0m[31m08:23:31,050 ERROR [io.undertow.request] (default task-70) UT005023: Exception handling request to /clusterbench/ejbservlet: java.lang.NullPointerException [JBossINF] at org.infinispan.transaction.impl.AbstractCacheTransaction.containsLockOrBackupLock(AbstractCacheTransaction.java:254) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.lambda$getTransactionWithLockedKey$177(DefaultPendingLockManager.java:252) [JBossINF] at org.infinispan.commons.util.concurrent.jdk8backported.EquivalentConcurrentHashMapV8$ValuesView.forEach(EquivalentConcurrentHashMapV8.java:4604) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.forEachTransaction(DefaultPendingLockManager.java:284) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.getTransactionWithLockedKey(DefaultPendingLockManager.java:249) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.checkForPendingLock(DefaultPendingLockManager.java:187) [JBossINF] at org.infinispan.util.concurrent.locks.impl.DefaultPendingLockManager.awaitPendingTransactionsForKey(DefaultPendingLockManager.java:115) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.checkPendingAndLockKey(AbstractTxLockingInterceptor.java:190) [JBossINF] at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.lockOrRegisterBackupLock(AbstractTxLockingInterceptor.java:115) [JBossINF] at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitDataReadCommand(PessimisticLockingInterceptor.java:69) [JBossINF] at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitGetKeyValueCommand(AbstractLockingInterceptor.java:76) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:85) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.TxInterceptor.enlistReadAndInvokeNext(TxInterceptor.java:345) [JBossINF] at org.infinispan.interceptors.TxInterceptor.visitGetKeyValueCommand(TxInterceptor.java:330) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:85) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:403) [JBossINF] at org.infinispan.statetransfer.StateTransferInterceptor.handleDefault(StateTransferInterceptor.java:388) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:85) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:102) [JBossINF] at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:71) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:85) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99) [JBossINF] at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113) [JBossINF] at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:85) [JBossINF] at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) [JBossINF] at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336) [JBossINF] at org.infinispan.cache.impl.CacheImpl.get(CacheImpl.java:412) [JBossINF] at org.infinispan.cache.impl.DecoratedCache.get(DecoratedCache.java:437) [JBossINF] at org.infinispan.cache.impl.AbstractDelegatingCache.get(AbstractDelegatingCache.java:286) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionMetaDataFactory.getValue(InfinispanSessionMetaDataFactory.java:70) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionMetaDataFactory.findValue(InfinispanSessionMetaDataFactory.java:60) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionMetaDataFactory.findValue(InfinispanSessionMetaDataFactory.java:36) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionFactory.findValue(InfinispanSessionFactory.java:61) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionFactory.findValue(InfinispanSessionFactory.java:40) [JBossINF] at org.wildfly.clustering.web.infinispan.session.InfinispanSessionManager.findSession(InfinispanSessionManager.java:234) [JBossINF] at org.wildfly.clustering.web.undertow.session.DistributableSessionManager.getSession(DistributableSessionManager.java:140) [JBossINF] at io.undertow.servlet.spec.ServletContextImpl.getSession(ServletContextImpl.java:725) [JBossINF] at io.undertow.servlet.spec.HttpServletRequestImpl.getSession(HttpServletRequestImpl.java:370) [JBossINF] at org.jboss.weld.servlet.SessionHolder.requestInitialized(SessionHolder.java:47) [JBossINF] at org.jboss.weld.servlet.HttpContextLifecycle.requestInitialized(HttpContextLifecycle.java:231) [JBossINF] at org.jboss.weld.servlet.WeldInitialListener.requestInitialized(WeldInitialListener.java:152) [JBossINF] at io.undertow.servlet.core.ApplicationListeners.requestInitialized(ApplicationListeners.java:245) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:283) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:263) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:81) [JBossINF] at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:174) [JBossINF] at io.undertow.server.Connectors.executeRootHandler(Connectors.java:202) [JBossINF] at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:793) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) {code} Link: http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-7x-failover-ejb-e... This issue may be related: https://issues.jboss.org/browse/WFLY-5208