[infinispan-issues] [JBoss JIRA] (ISPN-11030) ConcurrentModificationException - ArrayList$Itr.checkForComodification(ArrayList.java:1042) - regression-cs-rest-dist-(reads|writes)

ConcurrentModificationException - ArrayList$Itr.checkForComodification(ArrayList.java:1042) - regression-cs-rest-dist-(reads|writes) ------------------------------------------------------------------------------------------------------------------------------------ Key: ISPN-11030 URL: https://issues.jboss.org/browse/ISPN-11030 Project: Infinispan Issue Type: Bug Components: REST Affects Versions: 10.1.0.CR1 Environment: 10 clients, each client with 10 threads Reporter: Diego Lovison Assignee: Gustavo Fernandes Priority: Critical {noformat} 19:08:46,465 INFO [org.infinispan.CLUSTER] (remote-thread--p3-t2) [Context=rest]ISPN100002: Starting rebalance with members [edg-perf06-12311, edg-perf04-40561], phase READ_OLD_WRITE_ALL, topology id 2 19:08:46,680 INFO [org.infinispan.CLUSTER] (remote-thread--p3-t3) [Context=rest]ISPN100009: Advancing to rebalance phase READ_ALL_WRITE_ALL, topology id 3 19:08:46,718 INFO [org.infinispan.CLUSTER] (remote-thread--p3-t13) [Context=rest]ISPN100009: Advancing to rebalance phase READ_NEW_WRITE_ALL, topology id 4 19:08:46,607 ERROR [org.infinispan.rest.RestRequestHandler] (SINGLE_PORT-ServerIO-8-9) ISPN012006: Uncaught exception in the pipeline: java.util.ConcurrentModificationException at java.base/java.util.ArrayList$Itr.checkForComodification(ArrayList.java:1042) at java.base/java.util.ArrayList$Itr.next(ArrayList.java:996) at io.netty.handler.codec.http.cors.CorsHandler.getForOrigin(CorsHandler.java:125) at io.netty.handler.codec.http.cors.CorsHandler.channelRead(CorsHandler.java:85) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:326) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:300) at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:93) at org.infinispan.server.core.transport.StatsChannelHandler.channelRead(StatsChannelHandler.java:26) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931) at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:792) at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:502) at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:834) 19:08:46,791 INFO [org.infinispan.CLUSTER] (remote-thread--p3-t5) [Context=rest]ISPN100010: Finished rebalance with members [edg-perf06-12311, edg-perf04-40561], topology id 5 19:08:46,804 INFO [org.infinispan.CLUSTER] (remote-thread--p3-t5) [Context=rest]ISPN100002: Starting rebalance with members [edg-perf06-12311, edg-perf04-40561, edg-perf03-42053, edg-perf07-41186, edg-perf05-21163, edg-perf02-59957, edg-perf08-16062, edg-perf01-57801], phase READ_OLD_WRITE_ALL, topology id 6 19:08:46,949 INFO [org.infinispan.CLUSTER] (remote-thread--p3-t3) [Context=rest]ISPN100009: Advancing to rebalance phase READ_ALL_WRITE_ALL, topology id 7 19:08:47,014 INFO [org.infinispan.CLUSTER] (remote-thread--p3-t3) [Context=rest]ISPN100009: Advancing to rebalance phase READ_NEW_WRITE_ALL, topology id 8 19:08:47,023 INFO [org.infinispan.CLUSTER] (remote-thread--p3-t5) [Context=rest]ISPN100010: Finished rebalance with members [edg-perf06-12311, edg-perf04-40561, edg-perf03-42053, edg-perf07-41186, edg-perf05-21163, edg-perf02-59957, edg-perf08-16062, edg-perf01-57801], topology id 9 19:08:47,143 INFO [org.radargun.service.InfinispanRestAPI] (pool-2-thread-1) CacheManagerInfo{clusterMembers=[edg-perf01-57801, edg-perf03-42053, edg-perf02-59957, edg-perf04-40561, edg-perf05-21163, edg-perf07-41186, edg-perf06-12311, edg-perf08-16062], clusterSize=8} [40.227s][info ][gc] GC(6) Pause Young (Normal) (G1 Evacuation Pause) 838M->143M(16384M) 95.766ms 19:08:56,735 INFO [org.radargun.service.InfinispanRestAPI] (pool-2-thread-1) CacheManagerInfo{clusterMembers=[edg-perf01-57801, edg-perf03-42053, edg-perf02-59957, edg-perf04-40561, edg-perf05-21163, edg-perf07-41186, edg-perf06-12311, edg-perf08-16062], clusterSize=8} [45.080s][info ][gc] GC(7) Pause Young (Normal) (G1 Evacuation Pause) 855M->246M(16384M) 132.612ms [49.699s][info ][gc] GC(8) Pause Young (Normal) (G1 Evacuation Pause) 958M->348M(16384M) 116.067ms 19:09:06,731 INFO [org.radargun.service.InfinispanRestAPI] (pool-2-thread-1) CacheManagerInfo{clusterMembers=[edg-perf01-57801, edg-perf03-42053, edg-perf02-59957, edg-perf04-40561, edg-perf05-21163, edg-perf07-41186, edg-perf06-12311, edg-perf08-16062], clusterSize=8} [53.220s][info ][gc] GC(9) Pause Young (Normal) (G1 Evacuation Pause) 1060M->459M(16384M) 100.669ms [56.705s][info ][gc] GC(10) Pause Young (Normal) (G1 Evacuation Pause) 1171M->578M(16384M) 112.650ms 19:09:10,384 ERROR [org.infinispan.rest.RestRequestHandler] (SINGLE_PORT-ServerIO-8-15) ISPN012006: Uncaught exception in the pipeline: java.util.ConcurrentModificationException at java.base/java.util.ArrayList$Itr.checkForComodification(ArrayList.java:1042) at java.base/java.util.ArrayList$Itr.next(ArrayList.java:996) at io.netty.handler.codec.http.cors.CorsHandler.getForOrigin(CorsHandler.java:125) at io.netty.handler.codec.http.cors.CorsHandler.channelRead(CorsHandler.java:85) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:326) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:300) at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:93) at org.infinispan.server.core.transport.StatsChannelHandler.channelRead(StatsChannelHandler.java:26) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931) at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:792) at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:502) at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:834) [60.055s][info ][gc] GC(11) Pause Young (Normal) (G1 Evacuation Pause) 1290M->693M(16384M) 109.505ms 19:09:16,737 INFO [org.radargun.service.InfinispanRestAPI] (pool-2-thread-1) CacheManagerInfo{clusterMembers=[edg-perf01-57801, edg-perf03-42053, edg-perf02-59957, edg-perf04-40561, edg-perf05-21163, edg-perf07-41186, edg-perf06-12311, edg-perf08-16062], clusterSize=8} [63.456s][info ][gc] GC(12) Pause Young (Normal) (G1 Evacuation Pause) 1405M->807M(16384M) 109.216ms 19:09:17,139 ERROR [org.infinispan.rest.RestRequestHandler] (SINGLE_PORT-ServerIO-8-7) ISPN012006: Uncaught exception in the pipeline: java.util.ConcurrentModificationException at java.base/java.util.ArrayList$Itr.checkForComodification(ArrayList.java:1042) at java.base/java.util.ArrayList$Itr.next(ArrayList.java:996) at io.netty.handler.codec.http.cors.CorsHandler.getForOrigin(CorsHandler.java:125) at io.netty.handler.codec.http.cors.CorsHandler.channelRead(CorsHandler.java:85) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:326) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:300) at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:93) at org.infinispan.server.core.transport.StatsChannelHandler.channelRead(StatsChannelHandler.java:26) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931) at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:792) at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:502) at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:834) [66.872s][info ][gc] GC(13) Pause Young (Normal) (G1 Evacuation Pause) 1519M->925M(16384M) 105.106ms [70.354s][info ][gc] GC(14) Pause Young (Normal) (G1 Evacuation Pause) 1653M->1048M(16384M) 112.962ms 19:09:26,750 INFO [org.radargun.service.InfinispanRestAPI] (pool-2-thread-1) CacheManagerInfo{clusterMembers=[edg-perf01-57801, edg-perf03-42053, edg-perf02-59957, edg-perf04-40561, edg-perf05-21163, edg-perf07-41186, edg-perf06-12311, edg-perf08-16062], clusterSize=8} [74.102s][info ][gc] GC(15) Pause Young (Normal) (G1 Evacuation Pause) 1824M->1176M(16384M) 118.885ms 19:09:27,781 ERROR [org.infinispan.rest.RestRequestHandler] (SINGLE_PORT-ServerIO-8-1) ISPN012006: Uncaught exception in the pipeline: java.util.ConcurrentModificationException at java.base/java.util.ArrayList$Itr.checkForComodification(ArrayList.java:1042) {noformat}