Problem with distributable sessions on long IO ---------------------------------------------- Key: WFLY-4616 URL: https://issues.jboss.org/browse/WFLY-4616 Project: WildFly Issue Type: Bug Components: Clustering Affects Versions: 8.1.0.Final, 8.2.0.Final, 10.0.0.Final Reporter: Paweł Goździkowski Assignee: Paul Ferraro Priority: Critical Hi, I faced problem with wildfly <distributable/> what I want to ask is if it will be fixed/or is fixed in new versions on wildfly. The problem is when server starts performing long operation, ex. moving file to ntfs file system wildfly hangs over and starts to throw exceptions.. I tested it on wildfly 8.1 and 8.2 and wildfly and problem still occures. Disscusion about it you can find: https://developer.jboss.org/message/908454?et=watches.email.thread#908454 Exception: 13:19:00,670 ERROR [org.infinispan.interceptors.InvocationContextInterceptor] (default task-19) ISPN000136: Execution error: org.infinispan.util.concurrent.TimeoutException: Unable to acquire lock after [15 seconds] on key [j4-c64fI_msgIgF60Eqw_q72] for requestor [GlobalTransaction:<pgozdzik/web>:6832:local]! Lock held by [GlobalTransaction:<pgozdzik/web>:6830:local] at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:198) at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLock(LockManagerImpl.java:171) at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.lockKeyAndCheckOwnership(AbstractTxLockingInterceptor.java:169) at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitGetKeyValueCommand(PessimisticLockingInterceptor.java:70) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:74) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.TxInterceptor.enlistReadAndInvokeNext(TxInterceptor.java:226) at org.infinispan.interceptors.TxInterceptor.visitGetKeyValueCommand(TxInterceptor.java:221) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:74) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:263) at org.infinispan.statetransfer.StateTransferInterceptor.handleDefault(StateTransferInterceptor.java:247) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:74) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.CacheMgmtInterceptor.visitGetKeyValueCommand(CacheMgmtInterceptor.java:92) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:110) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:73) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:74) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:79) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:74) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:333) at org.infinispan.CacheImpl.get(CacheImpl.java:377) at org.infinispan.DecoratedCache.get(DecoratedCache.java:396) at org.infinispan.AbstractDelegatingCache.get(AbstractDelegatingCache.java:271) at org.jboss.as.clustering.infinispan.invoker.Locator$FindOperation.invoke(Locator.java:54) at org.jboss.as.clustering.infinispan.invoker.Locator$LockingFindOperation.invoke(Locator.java:71) at org.jboss.as.clustering.infinispan.invoker.SimpleCacheInvoker.invoke(SimpleCacheInvoker.java:34) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:82) at org.wildfly.clustering.web.infinispan.session.coarse.CoarseSessionFactory.findValue(CoarseSessionFactory.java:109) at org.wildfly.clustering.web.infinispan.session.coarse.CoarseSessionFactory.findValue(CoarseSessionFactory.java:55) at org.wildfly.clustering.web.infinispan.session.InfinispanSessionManager.findSession(InfinispanSessionManager.java:149) at org.wildfly.clustering.web.undertow.session.DistributableSessionManager.getSession(DistributableSessionManager.java:115) at io.undertow.servlet.spec.ServletContextImpl.getSession(ServletContextImpl.java:677) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.servlet.spec.HttpServletRequestImpl.getSession(HttpServletRequestImpl.java:353) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at org.jboss.weld.servlet.SessionHolder.requestInitialized(SessionHolder.java:47) [weld-core-impl-2.1.2.Final.jar:2014-01-09 09:23] at org.jboss.weld.servlet.HttpContextLifecycle.requestInitialized(HttpContextLifecycle.java:168) [weld-core-impl-2.1.2.Final.jar:2014-01-09 09:23] at org.jboss.weld.servlet.WeldInitialListener.requestInitialized(WeldInitialListener.java:153) [weld-core-impl-2.1.2.Final.jar:2014-01-09 09:23] at io.undertow.servlet.core.ApplicationListeners.requestInitialized(ApplicationListeners.java:216) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:239) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:227) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:73) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:146) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.server.Connectors.executeRootHandler(Connectors.java:177) [undertow-core-1.0.15.Final.jar:1.0.15.Final] at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:727) [undertow-core-1.0.15.Final.jar:1.0.15.Final] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [rt.jar:1.8.0_05] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [rt.jar:1.8.0_05] at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_05] 13:19:00,727 ERROR [io.undertow.request] (default task-19) UT005023: Exception handling request to /test: java.lang.IllegalStateException: Transaction DummyTransaction{xid=DummyXid{id=6832}, status=1} is not in a valid state to be invoking cache operations on. at org.infinispan.interceptors.TxInterceptor.enlist(TxInterceptor.java:275) at org.infinispan.interceptors.TxInterceptor.enlistIfNeeded(TxInterceptor.java:231) at org.infinispan.interceptors.TxInterceptor.enlistReadAndInvokeNext(TxInterceptor.java:225) at org.infinispan.interceptors.TxInterceptor.visitGetKeyValueCommand(TxInterceptor.java:221) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:112) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:74) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.statetransfer.StateTransferInterceptor.handleTopologyAffectedCommand(StateTransferInterceptor.java:263) at org.infinispan.statetransfer.StateTransferInterceptor.handleDefault(StateTransferInterceptor.java:247) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:74) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.CacheMgmtInterceptor.visitGetKeyValueCommand(CacheMgmtInterceptor.java:92) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:110) at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:73) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:74) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:98) at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:79) at org.infinispan.commands.AbstractVisitor.visitGetKeyValueCommand(AbstractVisitor.java:74) at org.infinispan.commands.read.GetKeyValueCommand.acceptVisitor(GetKeyValueCommand.java:40) at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:333) at org.infinispan.CacheImpl.get(CacheImpl.java:377) at org.infinispan.DecoratedCache.get(DecoratedCache.java:396) at org.infinispan.AbstractDelegatingCache.get(AbstractDelegatingCache.java:271) at org.jboss.as.clustering.infinispan.invoker.Locator$FindOperation.invoke(Locator.java:54) at org.jboss.as.clustering.infinispan.invoker.Locator$LockingFindOperation.invoke(Locator.java:71) at org.jboss.as.clustering.infinispan.invoker.SimpleCacheInvoker.invoke(SimpleCacheInvoker.java:34) at org.jboss.as.clustering.infinispan.invoker.RetryingCacheInvoker.invoke(RetryingCacheInvoker.java:82) at org.wildfly.clustering.web.infinispan.session.coarse.CoarseSessionFactory.findValue(CoarseSessionFactory.java:109) at org.wildfly.clustering.web.infinispan.session.coarse.CoarseSessionFactory.findValue(CoarseSessionFactory.java:55) at org.wildfly.clustering.web.infinispan.session.InfinispanSessionManager.findSession(InfinispanSessionManager.java:149) at org.wildfly.clustering.web.undertow.session.DistributableSessionManager.getSession(DistributableSessionManager.java:115) at io.undertow.servlet.spec.ServletContextImpl.getSession(ServletContextImpl.java:677) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.servlet.spec.HttpServletRequestImpl.getSession(HttpServletRequestImpl.java:353) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at org.jboss.weld.servlet.SessionHolder.requestInitialized(SessionHolder.java:47) [weld-core-impl-2.1.2.Final.jar:2014-01-09 09:23] at org.jboss.weld.servlet.HttpContextLifecycle.requestInitialized(HttpContextLifecycle.java:168) [weld-core-impl-2.1.2.Final.jar:2014-01-09 09:23] at org.jboss.weld.servlet.WeldInitialListener.requestInitialized(WeldInitialListener.java:153) [weld-core-impl-2.1.2.Final.jar:2014-01-09 09:23] at io.undertow.servlet.core.ApplicationListeners.requestInitialized(ApplicationListeners.java:216) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:239) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:227) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:73) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:146) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final] at io.undertow.server.Connectors.executeRootHandler(Connectors.java:177) [undertow-core-1.0.15.Final.jar:1.0.15.Final] at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:727) [undertow-core-1.0.15.Final.jar:1.0.15.Final] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [rt.jar:1.8.0_05] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [rt.jar:1.8.0_05] at java.lang.Thread.run(Thread.java:745) [rt.jar:1.8.0_05] - See more at: https://developer.jboss.org/message/908454?et=watches.email.thread#908454