[infinispan-issues] [JBoss JIRA] (ISPN-9867) Client listeners are removed when a notification occurs

Client listeners are removed when a notification occurs ------------------------------------------------------- Key: ISPN-9867 URL: https://issues.jboss.org/browse/ISPN-9867 Project: Infinispan Issue Type: Bug Components: Hot Rod, Listeners Affects Versions: 10.0.0.Alpha2, 9.4.5.Final Reporter: William Burns Assignee: William Burns Priority: Major Fix For: 10.0.0.Alpha3, 10.0.0.Final, 9.4.10.Final Client listeners are removed when a new notification occurs and they find that the channel is closed. This is problematic because this is done while notifying that object in general. This can cause issues if the listener is performing additional operations during this. For example Protobuf checks if an object passes a filter and then notifies it in one call. This will cause deadlocking issues since the filtering requires a read lock and the removal requires a write lock. An example of this can be seen in this stack trace: {code} "HotRod-ServerHandler-7-28" #121 prio=5 os_prio=0 tid=0x0000000003835000 nid=0x51a3 waiting on condition [0x00007f5967d6e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000000e1dd0058> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) at org.infinispan.objectfilter.impl.BaseMatcher.unregisterFilter(BaseMatcher.java:228) at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$Callback.unregister(BaseIckleFilterIndexingServiceProvider.java:198) at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$DelegatingCacheEntryListenerInvocationImpl.unregister(BaseIckleFilterIndexingServiceProvider.java:278) at org.infinispan.notifications.cachelistener.CacheNotifierImpl.removeListenerInvocation(CacheNotifierImpl.java:1895) at org.infinispan.notifications.impl.AbstractListenerImpl.removeListener(AbstractListenerImpl.java:148) at org.infinispan.notifications.cachelistener.CacheNotifierImpl.removeListener(CacheNotifierImpl.java:1874) at org.infinispan.cache.impl.CacheImpl.removeListener(CacheImpl.java:985) at org.infinispan.cache.impl.AbstractDelegatingCache.removeListener(AbstractDelegatingCache.java:550) at org.infinispan.server.hotrod.ClientListenerRegistry$BaseClientEventSender.isSendEvent(ClientListenerRegistry.java:367) at org.infinispan.server.hotrod.ClientListenerRegistry$BaseClientEventSender.onCacheEvent(ClientListenerRegistry.java:350) at sun.reflect.GeneratedMethodAccessor9.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl.lambda$invoke$1(AbstractListenerImpl.java:382) at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl$$Lambda$388/543782975.run(Unknown Source) at org.infinispan.util.concurrent.WithinThreadExecutor.execute(WithinThreadExecutor.java:20) at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl.invoke(AbstractListenerImpl.java:404) at org.infinispan.notifications.cachelistener.CacheNotifierImpl$BaseCacheEntryListenerInvocation.doRealInvocation(CacheNotifierImpl.java:1689) at org.infinispan.notifications.cachelistener.CacheNotifierImpl$ClusteredListenerInvocation.doRealInvocation(CacheNotifierImpl.java:1586) at org.infinispan.notifications.cachelistener.CacheNotifierImpl$BaseCacheEntryListenerInvocation.invokeNoChecks(CacheNotifierImpl.java:1680) at org.infinispan.notifications.cachelistener.filter.DelegatingCacheEntryListenerInvocation.invokeNoChecks(DelegatingCacheEntryListenerInvocation.java:54) at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$Callback.onFilterResult(BaseIckleFilterIndexingServiceProvider.java:261) at org.infinispan.objectfilter.impl.predicateindex.MatcherEvalContext.notifyDeltaSubscribers(MatcherEvalContext.java:184) at org.infinispan.objectfilter.impl.BaseMatcher.matchDelta(BaseMatcher.java:113) at org.infinispan.query.continuous.impl.IckleContinuousQueryFilterIndexingServiceProvider.matchEvent(IckleContinuousQueryFilterIndexingServiceProvider.java:85) at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$FilteringListenerInvocation.invoke(BaseIckleFilterIndexingServiceProvider.java:306) at org.infinispan.notifications.cachelistener.CacheNotifierImpl.notifyCacheEntryModified(CacheNotifierImpl.java:421) at org.infinispan.notifications.cachelistener.NotifyHelper.entryCommitted(NotifyHelper.java:67) at org.infinispan.interceptors.locking.ClusteringDependentLogic$LocalLogic.commitSingleEntry(ClusteringDependentLogic.java:362) at org.infinispan.interceptors.locking.ClusteringDependentLogic$AbstractClusteringDependentLogic.commitEntry(ClusteringDependentLogic.java:190) at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitContextEntry(EntryWrappingInterceptor.java:584) at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitEntryIfNeeded(EntryWrappingInterceptor.java:813) at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitContextEntries(EntryWrappingInterceptor.java:566) at org.infinispan.interceptors.impl.EntryWrappingInterceptor.applyChanges(EntryWrappingInterceptor.java:617) at org.infinispan.interceptors.impl.EntryWrappingInterceptor.applyAndFixVersion(EntryWrappingInterceptor.java:678) at org.infinispan.interceptors.impl.EntryWrappingInterceptor$$Lambda$379/274655666.accept(Unknown Source) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenAccept(BaseAsyncInterceptor.java:105) at org.infinispan.interceptors.impl.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:672) at org.infinispan.interceptors.impl.EntryWrappingInterceptor.visitPutKeyValueCommand(EntryWrappingInterceptor.java:302) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:150) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lambda$nonTxLockAndInvokeNext$1(AbstractLockingInterceptor.java:299) at org.infinispan.interceptors.locking.AbstractLockingInterceptor$$Lambda$554/57968464.apply(Unknown Source) at org.infinispan.interceptors.SyncInvocationStage.addCallback(SyncInvocationStage.java:42) at org.infinispan.interceptors.InvocationStage.andHandle(InvocationStage.java:65) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.nonTxLockAndInvokeNext(AbstractLockingInterceptor.java:294) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:126) at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitDataWriteCommand(NonTransactionalLockingInterceptor.java:40) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitPutKeyValueCommand(AbstractLockingInterceptor.java:82) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:150) at org.infinispan.interceptors.impl.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:217) at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:179) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54) at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53) at org.infinispan.interceptors.DDAsyncInterceptor.visitPutKeyValueCommand(DDAsyncInterceptor.java:59) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:123) at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90) at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:234) at org.infinispan.cache.impl.CacheImpl.executeCommandAndCommitIfNeededAsync(CacheImpl.java:1930) at org.infinispan.cache.impl.CacheImpl.putAsync(CacheImpl.java:1571) at org.infinispan.cache.impl.DecoratedCache.putAsync(DecoratedCache.java:690) at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.putAsync(AbstractDelegatingAdvancedCache.java:386) at org.infinispan.cache.impl.EncoderCache.putAsync(EncoderCache.java:460) at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.putAsync(AbstractDelegatingAdvancedCache.java:386) at org.infinispan.server.hotrod.CacheRequestProcessor.putInternal(CacheRequestProcessor.java:194) at org.infinispan.server.hotrod.CacheRequestProcessor.lambda$put$6(CacheRequestProcessor.java:187) at org.infinispan.server.hotrod.CacheRequestProcessor$$Lambda$535/1532854422.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) {code}