[infinispan-issues] [JBoss JIRA] (ISPN-7069) OOM on PutKeyValueCommand in invalidation mode with Redis cache store

[ https://issues.jboss.org/browse/ISPN-7069?page=com.atlassian.jira.plugin.... ] Vladimir Dzhuvinov updated ISPN-7069: ------------------------------------- Description: We've been working on isolating the cause of an exception we got last week with a Redis cache store (reported here [1]) when we started getting this OOM exception in the automated test trying to reproduce the exception: {{3217 [ERROR] InvocationContextInterceptor: ISPN000136: Error executing command PutKeyValueCommand, writing keys [["f-P6nC1W28lLgXGzg6YZY5emzxvIjhsrAJjnpz9MIzE",1]] org.infinispan.remoting.RemoteException: ISPN000217: Received exception from reload-43448, see cause for remote stack trace at org.infinispan.remoting.transport.AbstractTransport.checkResponse(AbstractTransport.java:44) ~[infinispan-core-8.2.4.Final.jar:8.2.4.Final] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:795) ~[infinispan-core-8.2.4.Final.jar:8.2.4.Final] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:642) ~[infinispan-core-8.2.4.Final.jar:8.2.4.Final] at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602) ~[?:1.8.0_91] at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) ~[?:1.8.0_91] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_91] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) ~[?:1.8.0_91] at org.infinispan.remoting.transport.jgroups.RspListFuture.futureDone(RspListFuture.java:31) ~[infinispan-core-8.2.4.Final.jar:8.2.4.Final] at org.jgroups.blocks.Request.checkCompletion(Request.java:162) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.GroupRequest.receiveResponse(GroupRequest.java:136) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:373) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:237) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:695) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.JChannel.up(JChannel.java:738) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1030) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:392) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:381) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1043) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1064) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:779) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:652) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:155) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:200) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:310) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.MERGE3.up(MERGE3.java:285) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.Discovery.up(Discovery.java:296) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.TP.passMessageUp(TP.java:1590) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1802) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_91] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_91] at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_91] Caused by: org.infinispan.commons.CacheException: Problems invoking command. at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:189) ~[infinispan-core-8.2.4.Final.jar:8.2.4.Final] at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:402) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:352) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:237) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:695) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.JChannel.up(JChannel.java:738) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1030) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:381) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:392) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1043) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:435) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.NAKACK2.deliver(NAKACK2.java:961) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.NAKACK2.handleMessage(NAKACK2.java:843) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:618) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] ... 10 more Caused by: java.lang.OutOfMemoryError: Java heap space}} The cache is configured in invalidation mode, with eviction enabled. The OOM exception occurs immediately on cache start. The Redis store is empty, and preload is disabled. [1] https://github.com/infinispan/infinispan-cachestore-redis/issues/2 was: We've been working on isolating the cause of an exception we got last week with a Redis cache store (reported here [1]) when we started getting this OOM exception in the automated test: {{3217 [ERROR] InvocationContextInterceptor: ISPN000136: Error executing command PutKeyValueCommand, writing keys [["f-P6nC1W28lLgXGzg6YZY5emzxvIjhsrAJjnpz9MIzE",1]] org.infinispan.remoting.RemoteException: ISPN000217: Received exception from reload-43448, see cause for remote stack trace at org.infinispan.remoting.transport.AbstractTransport.checkResponse(AbstractTransport.java:44) ~[infinispan-core-8.2.4.Final.jar:8.2.4.Final] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:795) ~[infinispan-core-8.2.4.Final.jar:8.2.4.Final] at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:642) ~[infinispan-core-8.2.4.Final.jar:8.2.4.Final] at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602) ~[?:1.8.0_91] at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) ~[?:1.8.0_91] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_91] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) ~[?:1.8.0_91] at org.infinispan.remoting.transport.jgroups.RspListFuture.futureDone(RspListFuture.java:31) ~[infinispan-core-8.2.4.Final.jar:8.2.4.Final] at org.jgroups.blocks.Request.checkCompletion(Request.java:162) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.GroupRequest.receiveResponse(GroupRequest.java:136) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:373) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:237) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:695) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.JChannel.up(JChannel.java:738) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1030) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:392) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:381) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1043) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1064) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:779) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:652) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:155) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:200) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:310) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.MERGE3.up(MERGE3.java:285) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.Discovery.up(Discovery.java:296) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.TP.passMessageUp(TP.java:1590) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1802) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_91] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_91] at java.lang.Thread.run(Thread.java:745) ~[?:1.8.0_91] Caused by: org.infinispan.commons.CacheException: Problems invoking command. at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:189) ~[infinispan-core-8.2.4.Final.jar:8.2.4.Final] at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:402) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:352) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:237) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:695) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.JChannel.up(JChannel.java:738) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1030) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FRAG2.up(FRAG2.java:165) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:381) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:392) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1043) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:435) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.NAKACK2.deliver(NAKACK2.java:961) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.NAKACK2.handleMessage(NAKACK2.java:843) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:618) ~[jgroups-3.6.7.Final.jar:3.6.7.Final] ... 10 more Caused by: java.lang.OutOfMemoryError: Java heap space}} The cache is configured in invalidation mode, with eviction enabled. The OOM exception occurs immediately on cache start. The Redis store is empty, and preload is disabled. [1] https://github.com/infinispan/infinispan-cachestore-redis/issues/2 -- This message was sent by Atlassian JIRA (v6.4.11#64026)