HeuristicMixedException in simple failover scenario --------------------------------------------------- Key: ISPN-6862 URL: https://issues.jboss.org/browse/ISPN-6862 Project: Infinispan Issue Type: Bug Affects Versions: 8.1.4.Final Reporter: Bogdan Sikora https://paste.fedoraproject.org/391436/ Scenario: * 3x Eap as worker, 1x apache core as Balancer * Send request to balancer * Kill worker that handled request * Send request to balancer {noformat} 2016-07-15 04:51:51,916 ERROR [stderr] (ServerService Thread Pool -- 82) Exception in thread "ServerService Thread Pool -- 82" org.infinispan.commons.CacheException: javax.transaction.HeuristicMixedException 2016-07-15 04:51:51,917 ERROR [stderr] (ServerService Thread Pool -- 82) at org.wildfly.clustering.ee.infinispan.InfinispanBatch.close(InfinispanBatch.java:74) 2016-07-15 04:51:51,919 ERROR [stderr] (ServerService Thread Pool -- 82) at org.wildfly.clustering.server.registry.CacheRegistry.lambda$close$0(CacheRegistry.java:101) 2016-07-15 04:51:51,919 ERROR [stderr] (ServerService Thread Pool -- 82) at org.wildfly.clustering.server.registry.CacheRegistry$$Lambda$196.00000000E4013240.run(Unknown Source) 2016-07-15 04:51:51,920 ERROR [stderr] (ServerService Thread Pool -- 82) at org.wildfly.clustering.service.concurrent.StampedLockServiceExecutor.close(StampedLockServiceExecutor.java:55) 2016-07-15 04:51:51,921 ERROR [stderr] (ServerService Thread Pool -- 82) at org.wildfly.clustering.server.registry.CacheRegistry.close(CacheRegistry.java:94) 2016-07-15 04:51:51,921 INFO [org.jboss.as.clustering.infinispan] (ServerService Thread Pool -- 91) WFLYCLINF0003: Stopped routing cache from web container 2016-07-15 04:51:51,922 ERROR [stderr] (ServerService Thread Pool -- 82) at org.wildfly.clustering.server.registry.CacheRegistryFactory$1.close(CacheRegistryFactory.java:56) 2016-07-15 04:51:51,924 ERROR [stderr] (ServerService Thread Pool -- 82) at org.wildfly.clustering.server.registry.RegistryBuilder.stop(RegistryBuilder.java:88) 2016-07-15 04:51:51,925 INFO [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-1) ISPN000080: Disconnecting JGroups channel web 2016-07-15 04:51:51,926 ERROR [stderr] (ServerService Thread Pool -- 82) at org.wildfly.clustering.service.AsynchronousServiceBuilder$2.run(AsynchronousServiceBuilder.java:130) 2016-07-15 04:51:51,927 ERROR [stderr] (ServerService Thread Pool -- 82) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1153) 2016-07-15 04:51:51,927 ERROR [stderr] (ServerService Thread Pool -- 82) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) 2016-07-15 04:51:51,927 WARN [org.infinispan.topology.ClusterTopologyManagerImpl] (transport-thread--p12-t6) ISPN000197: Error updating cluster member list: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from jboss-eap-7.0, see cause for remote stack trace at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:764) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$0(JGroupsTransport.java:602) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$$Lambda$169.00000000F91C85F0.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:613) at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:588) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:485) at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1973) at org.infinispan.remoting.transport.jgroups.SingleResponseFuture.futureDone(SingleResponseFuture.java:30) at org.jgroups.blocks.Request.checkCompletion(Request.java:151) at org.jgroups.blocks.UnicastRequest.transportClosed(UnicastRequest.java:169) at org.jgroups.blocks.RequestCorrelator.stop(RequestCorrelator.java:253) at org.jgroups.blocks.MessageDispatcher.stop(MessageDispatcher.java:167) at org.jgroups.blocks.MessageDispatcher.channelDisconnected(MessageDispatcher.java:537) at org.jgroups.Channel.notifyChannelDisconnected(Channel.java:533) at org.jgroups.fork.ForkChannel.disconnect(ForkChannel.java:186) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.stop(JGroupsTransport.java:263) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:95) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:55) at java.lang.reflect.Method.invoke(Method.java:508) at org.infinispan.commons.util.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:168) at org.infinispan.factories.AbstractComponentRegistry$PrioritizedMethod.invoke(AbstractComponentRegistry.java:887) at org.infinispan.factories.AbstractComponentRegistry.internalStop(AbstractComponentRegistry.java:692) at org.infinispan.factories.AbstractComponentRegistry.stop(AbstractComponentRegistry.java:570) at org.infinispan.factories.GlobalComponentRegistry.stop(GlobalComponentRegistry.java:263) at org.infinispan.manager.DefaultCacheManager.stop(DefaultCacheManager.java:689) at org.jboss.as.clustering.infinispan.subsystem.CacheContainerBuilder.stop(CacheContainerBuilder.java:120) at org.jboss.msc.service.ServiceControllerImpl$StopTask.stopService(ServiceControllerImpl.java:2056) at org.jboss.msc.service.ServiceControllerImpl$StopTask.run(ServiceControllerImpl.java:2017) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1153) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.lang.Thread.run(Thread.java:785) Caused by: java.lang.IllegalStateException: transport was closed at org.jgroups.blocks.UnicastRequest.transportClosed(UnicastRequest.java:164) ... 22 more 2016-07-15 04:51:51,928 ERROR [stderr] (ServerService Thread Pool -- 82) at java.lang.Thread.run(Thread.java:785) 2016-07-15 04:51:51,929 ERROR [stderr] (ServerService Thread Pool -- 82) at org.jboss.threads.JBossThread.run(JBossThread.java:320) 2016-07-15 04:51:51,930 WARN [org.infinispan.topology.ClusterTopologyManagerImpl] (transport-thread--p12-t6) ISPN000197: Error updating cluster member list: org.infinispan.commons.CacheException: java.lang.IllegalStateException: channel is not connected at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.rethrowAsCacheException(CommandAwareRpcDispatcher.java:132) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommands(CommandAwareRpcDispatcher.java:112) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotelyAsync(JGroupsTransport.java:551) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:505) at org.infinispan.topology.ClusterTopologyManagerImpl.confirmMembersAvailable(ClusterTopologyManagerImpl.java:465) at org.infinispan.topology.ClusterTopologyManagerImpl.updateCacheMembers(ClusterTopologyManagerImpl.java:452) at org.infinispan.topology.ClusterTopologyManagerImpl.handleClusterView(ClusterTopologyManagerImpl.java:363) at org.infinispan.topology.ClusterTopologyManagerImpl$ClusterViewListener.lambda$handleViewChange$0(ClusterTopologyManagerImpl.java:650) at org.infinispan.topology.ClusterTopologyManagerImpl$ClusterViewListener$$Lambda$167.000000009C90CF80.call(Unknown Source) at java.util.concurrent.FutureTask.run(FutureTask.java:277) at org.infinispan.executors.SemaphoreCompletionService$QueueingTask.runInternal(SemaphoreCompletionService.java:173) at org.infinispan.executors.SemaphoreCompletionService$QueueingTask.run(SemaphoreCompletionService.java:151) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1153) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at org.jboss.as.clustering.infinispan.ClassLoaderThreadFactory.lambda$newThread$0(ClassLoaderThreadFactory.java:48) at org.jboss.as.clustering.infinispan.ClassLoaderThreadFactory$$Lambda$129.00000000A04F9300.run(Unknown Source) at java.lang.Thread.run(Thread.java:785) Caused by: java.lang.IllegalStateException: channel is not connected at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.down(MessageDispatcher.java:713) at org.jgroups.blocks.RequestCorrelator.sendRequest(RequestCorrelator.java:164) at org.jgroups.blocks.GroupRequest.sendRequest(GroupRequest.java:325) at org.jgroups.blocks.GroupRequest.sendRequest(GroupRequest.java:76) at org.jgroups.blocks.Request.execute(Request.java:66) at org.jgroups.blocks.MessageDispatcher.cast(MessageDispatcher.java:363) at org.jgroups.blocks.MessageDispatcher.cast(MessageDispatcher.java:376) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processCalls(CommandAwareRpcDispatcher.java:300) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommands(CommandAwareRpcDispatcher.java:108) ... 15 more {noformat}