ClosedChannelException in SingleFileStore after server is shut down ------------------------------------------------------------------- Key: WFLY-5866 URL: https://issues.jboss.org/browse/WFLY-5866 Project: WildFly Issue Type: Bug Components: Clustering Affects Versions: 10.0.0.CR4 Reporter: Michal Vinkler Assignee: Paul Ferraro Seen in our failover tests for remote stateful EJBs: - scenario ejb-ejbremote-shutdown-dist-sync-tcpStack After perf21 was shut down, other nodes received new view and logged these exceptions, all of them having the same stacktrace and coming from {{org.infinispan.persistence.file.SingleFileStore._load(SingleFileStore.java:484)}} {code:title=AsynchronousCloseException} [JBossINF] [0m[31m18:06:00,890 ERROR [org.infinispan.persistence.file.SingleFileStore] (transport-thread--p16-t13) ISPN000252: Error executing parallel store task: org.infinispan.persistence.spi.PersistenceException: java.nio.channels.AsynchronousCloseException [JBossINF] at org.infinispan.persistence.file.SingleFileStore._load(SingleFileStore.java:484) [JBossINF] at org.infinispan.persistence.file.SingleFileStore.access$600(SingleFileStore.java:71) [JBossINF] at org.infinispan.persistence.file.SingleFileStore$2.call(SingleFileStore.java:535) [JBossINF] at org.infinispan.persistence.file.SingleFileStore$2.call(SingleFileStore.java:531) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at org.infinispan.util.concurrent.WithinThreadExecutor.execute(WithinThreadExecutor.java:21) [JBossINF] at java.util.concurrent.ExecutorCompletionService.submit(ExecutorCompletionService.java:181) [JBossINF] at org.infinispan.executors.ExecutorAllCompletionService.submit(ExecutorAllCompletionService.java:31) [JBossINF] at org.infinispan.persistence.file.SingleFileStore.process(SingleFileStore.java:531) [JBossINF] at org.infinispan.statetransfer.OutboundTransferTask.run(OutboundTransferTask.java:171) [JBossINF] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) [JBossINF] Caused by: java.nio.channels.AsynchronousCloseException [JBossINF] at java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:205) [JBossINF] at sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:745) [JBossINF] at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:726) [JBossINF] at org.infinispan.persistence.file.SingleFileStore._load(SingleFileStore.java:482) [JBossINF] ... 18 more {code} {code:title=ClosedByInterruptException} [JBossINF] [0m[31m18:06:00,890 ERROR [org.infinispan.persistence.file.SingleFileStore] (transport-thread--p16-t10) ISPN000252: Error executing parallel store task: org.infinispan.persistence.spi.PersistenceException: java.nio.channels.ClosedByInterruptException [JBossINF] at org.infinispan.persistence.file.SingleFileStore._load(SingleFileStore.java:484) [JBossINF] at org.infinispan.persistence.file.SingleFileStore.access$600(SingleFileStore.java:71) [JBossINF] at org.infinispan.persistence.file.SingleFileStore$2.call(SingleFileStore.java:535) [JBossINF] at org.infinispan.persistence.file.SingleFileStore$2.call(SingleFileStore.java:531) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at org.infinispan.util.concurrent.WithinThreadExecutor.execute(WithinThreadExecutor.java:21) [JBossINF] at java.util.concurrent.ExecutorCompletionService.submit(ExecutorCompletionService.java:181) [JBossINF] at org.infinispan.executors.ExecutorAllCompletionService.submit(ExecutorAllCompletionService.java:31) [JBossINF] at org.infinispan.persistence.file.SingleFileStore.process(SingleFileStore.java:531) [JBossINF] at org.infinispan.statetransfer.OutboundTransferTask.run(OutboundTransferTask.java:171) [JBossINF] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) [JBossINF] Caused by: java.nio.channels.ClosedByInterruptException [JBossINF] at java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202) [JBossINF] at sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:745) [JBossINF] at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:726) [JBossINF] at org.infinispan.persistence.file.SingleFileStore._load(SingleFileStore.java:482) [JBossINF] ... 18 more {code} {code:title=ClosedChannelException} [JBossINF] [0m[31m18:06:00,895 ERROR [org.infinispan.persistence.file.SingleFileStore] (transport-thread--p16-t13) ISPN000252: Error executing parallel store task: org.infinispan.persistence.spi.PersistenceException: java.nio.channels.ClosedChannelException [JBossINF] at org.infinispan.persistence.file.SingleFileStore._load(SingleFileStore.java:484) [JBossINF] at org.infinispan.persistence.file.SingleFileStore.access$600(SingleFileStore.java:71) [JBossINF] at org.infinispan.persistence.file.SingleFileStore$2.call(SingleFileStore.java:535) [JBossINF] at org.infinispan.persistence.file.SingleFileStore$2.call(SingleFileStore.java:531) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at org.infinispan.util.concurrent.WithinThreadExecutor.execute(WithinThreadExecutor.java:21) [JBossINF] at java.util.concurrent.ExecutorCompletionService.submit(ExecutorCompletionService.java:181) [JBossINF] at org.infinispan.executors.ExecutorAllCompletionService.submit(ExecutorAllCompletionService.java:31) [JBossINF] at org.infinispan.persistence.file.SingleFileStore.process(SingleFileStore.java:531) [JBossINF] at org.infinispan.statetransfer.OutboundTransferTask.run(OutboundTransferTask.java:171) [JBossINF] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [JBossINF] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) [JBossINF] Caused by: java.nio.channels.ClosedChannelException [JBossINF] at sun.nio.ch.FileChannelImpl.ensureOpen(FileChannelImpl.java:110) [JBossINF] at sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:720) [JBossINF] at org.infinispan.persistence.file.SingleFileStore._load(SingleFileStore.java:482) [JBossINF] ... 18 more {code} While the first two exceptions were logged only few times, the {{ClosedChannelException}} was logged tens of thousand times on perf19 and perf20. These exceptions don't seem to cause any sampling errors on the client side. Link: http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/eap-7x-failover-ejb-e...