[JBoss JIRA] (ISPN-9871) Error executing command PutKeyValueCommand when using protobuf
by Diego Lovison (Jira)
Diego Lovison created ISPN-9871:
-----------------------------------
Summary: Error executing command PutKeyValueCommand when using protobuf
Key: ISPN-9871
URL: https://issues.jboss.org/browse/ISPN-9871
Project: Infinispan
Issue Type: Bug
Affects Versions: 9.4.5.Final
Reporter: Diego Lovison
{code:java}
private final String KEY = "key";
builder.addServer().host("127.0.0.1").port(11222).marshaller(new ProtoStreamMarshaller());
ProtobufHelper.register(cacheManager, "/library.proto");
RemoteCache cache = cacheManager.getCache("DEVICES");
cache.put(KEY + counter++, new Person("foo" + counter, counter));
{code}
Server1 - Stacktrace
{noformat}
10:34:07,799 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (jgroups-6,diegos-macbook-pro) ISPN000136: Error executing command PutKeyValueCommand, writing keys [WrappedByteArray{bytes=[B0x4A076B6579333339..[9], hashCode=1989086686}]: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from diegos-macbook-pro, see cause for remote stack trace
at org.infinispan.remoting.transport.ResponseCollectors.wrapRemoteException(ResponseCollectors.java:27)
at org.infinispan.remoting.transport.ValidSingleResponseCollector.withException(ValidSingleResponseCollector.java:37)
at org.infinispan.remoting.transport.ValidSingleResponseCollector.addResponse(ValidSingleResponseCollector.java:21)
at org.infinispan.remoting.transport.impl.SingleTargetRequest.receiveResponse(SingleTargetRequest.java:52)
at org.infinispan.remoting.transport.impl.SingleTargetRequest.onResponse(SingleTargetRequest.java:35)
at org.infinispan.remoting.transport.impl.RequestRepository.addResponse(RequestRepository.java:52)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processResponse(JGroupsTransport.java:1372)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1275)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$300(JGroupsTransport.java:126)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1420)
at org.jgroups.JChannel.up(JChannel.java:816)
at org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:133)
at org.jgroups.stack.Protocol.up(Protocol.java:340)
at org.jgroups.protocols.FORK.up(FORK.java:141)
at org.jgroups.protocols.FRAG3.up(FRAG3.java:171)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:343)
at org.jgroups.protocols.FlowControl.up(FlowControl.java:343)
at org.jgroups.protocols.pbcast.GMS.up(GMS.java:873)
at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:240)
at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1003)
at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:729)
at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:384)
at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:600)
at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:130)
at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:203)
at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:253)
at org.jgroups.protocols.MERGE3.up(MERGE3.java:280)
at org.jgroups.protocols.Discovery.up(Discovery.java:269)
at org.jgroups.protocols.TP.passMessageUp(TP.java:1248)
at org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:87)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.IllegalArgumentException: Descriptor not found : jira_verification.Person
at org.infinispan.protostream.impl.SerializationContextImpl.getDescriptorByName(SerializationContextImpl.java:334)
at org.infinispan.query.remote.impl.indexing.WrappedMessageTagHandler.onTag(WrappedMessageTagHandler.java:67)
at org.infinispan.protostream.ProtobufParser.parseMessage(ProtobufParser.java:75)
at org.infinispan.protostream.ProtobufParser.parseInternal(ProtobufParser.java:57)
at org.infinispan.protostream.ProtobufParser.parse(ProtobufParser.java:44)
at org.infinispan.query.remote.impl.indexing.ProtobufValueWrapperSearchWorkCreator.discoverMessageType(ProtobufValueWrapperSearchWorkCreator.java:90)
at org.infinispan.query.remote.impl.indexing.ProtobufValueWrapperSearchWorkCreator.interceptValue(ProtobufValueWrapperSearchWorkCreator.java:77)
at org.infinispan.query.remote.impl.indexing.ProtobufValueWrapperSearchWorkCreator.access$300(ProtobufValueWrapperSearchWorkCreator.java:27)
at org.infinispan.query.remote.impl.indexing.ProtobufValueWrapperSearchWorkCreator$Delegate.createPerEntityWorks(ProtobufValueWrapperSearchWorkCreator.java:57)
at org.infinispan.query.backend.QueryInterceptor.performSearchWork(QueryInterceptor.java:377)
at org.infinispan.query.backend.QueryInterceptor.updateIndexes(QueryInterceptor.java:372)
at org.infinispan.query.backend.QueryInterceptor.processChange(QueryInterceptor.java:455)
at org.infinispan.query.backend.QueryInterceptor.lambda$handleDataWriteCommand$0(QueryInterceptor.java:188)
at org.infinispan.interceptors.InvocationSuccessAction.apply(InvocationSuccessAction.java:22)
at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.invokeQueuedHandlers(QueueAsyncInvocationStage.java:118)
at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.accept(QueueAsyncInvocationStage.java:81)
at org.infinispan.interceptors.impl.QueueAsyncInvocationStage.accept(QueueAsyncInvocationStage.java:30)
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760)
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736)
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
at org.infinispan.remoting.transport.AbstractRequest.complete(AbstractRequest.java:67)
at org.infinispan.remoting.transport.impl.MultiTargetRequest.onResponse(MultiTargetRequest.java:102)
... 28 more
{noformat}
Server2
{noformat}
10:34:05,473 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (stateTransferExecutor-thread--p5-t38) ISPN000136: Error executing command PutKeyValueCommand, writing keys [/library.proto]: org.infinispan.commons.CacheException: ISPN028008: The value must be a String : class org.infinispan.container.entries.metadata.MetadataImmortalCacheEntry
at org.infinispan.query.remote.impl.ProtobufMetadataManagerInterceptor.visitPutKeyValueCommand(ProtobufMetadataManagerInterceptor.java:219)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:69)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.lambda$new$0(AbstractTxLockingInterceptor.java:34)
at org.infinispan.interceptors.InvocationSuccessFunction.apply(InvocationSuccessFunction.java:25)
at org.infinispan.interceptors.SyncInvocationStage.addCallback(SyncInvocationStage.java:42)
at org.infinispan.interceptors.InvocationStage.thenApply(InvocationStage.java:45)
at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.acquireLocalLockAndInvokeNext(PessimisticLockingInterceptor.java:283)
at org.infinispan.interceptors.locking.PessimisticLockingInterceptor.visitDataWriteCommand(PessimisticLockingInterceptor.java:148)
at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitPutKeyValueCommand(AbstractLockingInterceptor.java:82)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:69)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:150)
at org.infinispan.interceptors.impl.TxInterceptor.handleWriteCommand(TxInterceptor.java:402)
at org.infinispan.interceptors.impl.TxInterceptor.visitPutKeyValueCommand(TxInterceptor.java:235)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:69)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
at org.infinispan.statetransfer.TransactionSynchronizerInterceptor.visitCommand(TransactionSynchronizerInterceptor.java:41)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:185)
at org.infinispan.statetransfer.StateTransferInterceptor.handleTxWriteCommand(StateTransferInterceptor.java:260)
at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:250)
at org.infinispan.statetransfer.StateTransferInterceptor.visitPutKeyValueCommand(StateTransferInterceptor.java:96)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:69)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
at org.infinispan.interceptors.impl.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:214)
at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:179)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:69)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:123)
at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:56)
at org.infinispan.interceptors.impl.BatchingInterceptor.handleDefault(BatchingInterceptor.java:69)
at org.infinispan.interceptors.impl.BatchingInterceptor.visitPutKeyValueCommand(BatchingInterceptor.java:49)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:69)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.visitPutKeyValueCommand(DDAsyncInterceptor.java:60)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:69)
at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:50)
at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:248)
at org.infinispan.statetransfer.StateConsumerImpl.doApplyState(StateConsumerI
{noformat}
--
This message was sent by Atlassian Jira
(v7.12.1#712002)
3 years, 5 months
[JBoss JIRA] (ISPN-9867) Client listeners are removed when a notification occurs
by Galder Zamarreño (Jira)
[ https://issues.jboss.org/browse/ISPN-9867?page=com.atlassian.jira.plugin.... ]
Galder Zamarreño updated ISPN-9867:
-----------------------------------
Status: Resolved (was: Pull Request Sent)
Resolution: Done
> Client listeners are removed when a notification occurs
> -------------------------------------------------------
>
> Key: ISPN-9867
> URL: https://issues.jboss.org/browse/ISPN-9867
> Project: Infinispan
> Issue Type: Bug
> Components: Hot Rod, Listeners
> Affects Versions: 10.0.0.Alpha2, 9.4.5.Final
> Reporter: William Burns
> Assignee: William Burns
> Priority: Major
> Fix For: 10.0.0.Beta1, 10.0.0.Final
>
>
> Client listeners are removed when a new notification occurs and they find that the channel is closed. This is problematic because this is done while notifying that object in general. This can cause issues if the listener is performing additional operations during this. For example Protobuf checks if an object passes a filter and then notifies it in one call. This will cause deadlocking issues since the filtering requires a read lock and the removal requires a write lock.
> An example of this can be seen in this stack trace:
> {code}
> "HotRod-ServerHandler-7-28" #121 prio=5 os_prio=0 tid=0x0000000003835000 nid=0x51a3 waiting on condition [0x00007f5967d6e000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x00000000e1dd0058> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199)
> at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943)
> at org.infinispan.objectfilter.impl.BaseMatcher.unregisterFilter(BaseMatcher.java:228)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$Callback.unregister(BaseIckleFilterIndexingServiceProvider.java:198)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$DelegatingCacheEntryListenerInvocationImpl.unregister(BaseIckleFilterIndexingServiceProvider.java:278)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl.removeListenerInvocation(CacheNotifierImpl.java:1895)
> at org.infinispan.notifications.impl.AbstractListenerImpl.removeListener(AbstractListenerImpl.java:148)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl.removeListener(CacheNotifierImpl.java:1874)
> at org.infinispan.cache.impl.CacheImpl.removeListener(CacheImpl.java:985)
> at org.infinispan.cache.impl.AbstractDelegatingCache.removeListener(AbstractDelegatingCache.java:550)
> at org.infinispan.server.hotrod.ClientListenerRegistry$BaseClientEventSender.isSendEvent(ClientListenerRegistry.java:367)
> at org.infinispan.server.hotrod.ClientListenerRegistry$BaseClientEventSender.onCacheEvent(ClientListenerRegistry.java:350)
> at sun.reflect.GeneratedMethodAccessor9.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl.lambda$invoke$1(AbstractListenerImpl.java:382)
> at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl$$Lambda$388/543782975.run(Unknown Source)
> at org.infinispan.util.concurrent.WithinThreadExecutor.execute(WithinThreadExecutor.java:20)
> at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl.invoke(AbstractListenerImpl.java:404)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl$BaseCacheEntryListenerInvocation.doRealInvocation(CacheNotifierImpl.java:1689)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl$ClusteredListenerInvocation.doRealInvocation(CacheNotifierImpl.java:1586)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl$BaseCacheEntryListenerInvocation.invokeNoChecks(CacheNotifierImpl.java:1680)
> at org.infinispan.notifications.cachelistener.filter.DelegatingCacheEntryListenerInvocation.invokeNoChecks(DelegatingCacheEntryListenerInvocation.java:54)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$Callback.onFilterResult(BaseIckleFilterIndexingServiceProvider.java:261)
> at org.infinispan.objectfilter.impl.predicateindex.MatcherEvalContext.notifyDeltaSubscribers(MatcherEvalContext.java:184)
> at org.infinispan.objectfilter.impl.BaseMatcher.matchDelta(BaseMatcher.java:113)
> at org.infinispan.query.continuous.impl.IckleContinuousQueryFilterIndexingServiceProvider.matchEvent(IckleContinuousQueryFilterIndexingServiceProvider.java:85)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$FilteringListenerInvocation.invoke(BaseIckleFilterIndexingServiceProvider.java:306)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl.notifyCacheEntryModified(CacheNotifierImpl.java:421)
> at org.infinispan.notifications.cachelistener.NotifyHelper.entryCommitted(NotifyHelper.java:67)
> at org.infinispan.interceptors.locking.ClusteringDependentLogic$LocalLogic.commitSingleEntry(ClusteringDependentLogic.java:362)
> at org.infinispan.interceptors.locking.ClusteringDependentLogic$AbstractClusteringDependentLogic.commitEntry(ClusteringDependentLogic.java:190)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitContextEntry(EntryWrappingInterceptor.java:584)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitEntryIfNeeded(EntryWrappingInterceptor.java:813)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitContextEntries(EntryWrappingInterceptor.java:566)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.applyChanges(EntryWrappingInterceptor.java:617)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.applyAndFixVersion(EntryWrappingInterceptor.java:678)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor$$Lambda$379/274655666.accept(Unknown Source)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenAccept(BaseAsyncInterceptor.java:105)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:672)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.visitPutKeyValueCommand(EntryWrappingInterceptor.java:302)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:150)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lambda$nonTxLockAndInvokeNext$1(AbstractLockingInterceptor.java:299)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor$$Lambda$554/57968464.apply(Unknown Source)
> at org.infinispan.interceptors.SyncInvocationStage.addCallback(SyncInvocationStage.java:42)
> at org.infinispan.interceptors.InvocationStage.andHandle(InvocationStage.java:65)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.nonTxLockAndInvokeNext(AbstractLockingInterceptor.java:294)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:126)
> at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitDataWriteCommand(NonTransactionalLockingInterceptor.java:40)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitPutKeyValueCommand(AbstractLockingInterceptor.java:82)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:150)
> at org.infinispan.interceptors.impl.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:217)
> at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:179)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
> at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53)
> at org.infinispan.interceptors.DDAsyncInterceptor.visitPutKeyValueCommand(DDAsyncInterceptor.java:59)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:123)
> at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90)
> at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:234)
> at org.infinispan.cache.impl.CacheImpl.executeCommandAndCommitIfNeededAsync(CacheImpl.java:1930)
> at org.infinispan.cache.impl.CacheImpl.putAsync(CacheImpl.java:1571)
> at org.infinispan.cache.impl.DecoratedCache.putAsync(DecoratedCache.java:690)
> at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.putAsync(AbstractDelegatingAdvancedCache.java:386)
> at org.infinispan.cache.impl.EncoderCache.putAsync(EncoderCache.java:460)
> at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.putAsync(AbstractDelegatingAdvancedCache.java:386)
> at org.infinispan.server.hotrod.CacheRequestProcessor.putInternal(CacheRequestProcessor.java:194)
> at org.infinispan.server.hotrod.CacheRequestProcessor.lambda$put$6(CacheRequestProcessor.java:187)
> at org.infinispan.server.hotrod.CacheRequestProcessor$$Lambda$535/1532854422.run(Unknown Source)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
> at java.lang.Thread.run(Thread.java:748)
> {code}
--
This message was sent by Atlassian Jira
(v7.12.1#712002)
3 years, 5 months
[JBoss JIRA] (ISPN-9867) Client listeners are removed when a notification occurs
by Galder Zamarreño (Jira)
[ https://issues.jboss.org/browse/ISPN-9867?page=com.atlassian.jira.plugin.... ]
Galder Zamarreño updated ISPN-9867:
-----------------------------------
Fix Version/s: 10.0.0.Final
> Client listeners are removed when a notification occurs
> -------------------------------------------------------
>
> Key: ISPN-9867
> URL: https://issues.jboss.org/browse/ISPN-9867
> Project: Infinispan
> Issue Type: Bug
> Components: Hot Rod, Listeners
> Affects Versions: 10.0.0.Alpha2, 9.4.5.Final
> Reporter: William Burns
> Assignee: William Burns
> Priority: Major
> Fix For: 10.0.0.Beta1, 10.0.0.Final
>
>
> Client listeners are removed when a new notification occurs and they find that the channel is closed. This is problematic because this is done while notifying that object in general. This can cause issues if the listener is performing additional operations during this. For example Protobuf checks if an object passes a filter and then notifies it in one call. This will cause deadlocking issues since the filtering requires a read lock and the removal requires a write lock.
> An example of this can be seen in this stack trace:
> {code}
> "HotRod-ServerHandler-7-28" #121 prio=5 os_prio=0 tid=0x0000000003835000 nid=0x51a3 waiting on condition [0x00007f5967d6e000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x00000000e1dd0058> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199)
> at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943)
> at org.infinispan.objectfilter.impl.BaseMatcher.unregisterFilter(BaseMatcher.java:228)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$Callback.unregister(BaseIckleFilterIndexingServiceProvider.java:198)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$DelegatingCacheEntryListenerInvocationImpl.unregister(BaseIckleFilterIndexingServiceProvider.java:278)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl.removeListenerInvocation(CacheNotifierImpl.java:1895)
> at org.infinispan.notifications.impl.AbstractListenerImpl.removeListener(AbstractListenerImpl.java:148)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl.removeListener(CacheNotifierImpl.java:1874)
> at org.infinispan.cache.impl.CacheImpl.removeListener(CacheImpl.java:985)
> at org.infinispan.cache.impl.AbstractDelegatingCache.removeListener(AbstractDelegatingCache.java:550)
> at org.infinispan.server.hotrod.ClientListenerRegistry$BaseClientEventSender.isSendEvent(ClientListenerRegistry.java:367)
> at org.infinispan.server.hotrod.ClientListenerRegistry$BaseClientEventSender.onCacheEvent(ClientListenerRegistry.java:350)
> at sun.reflect.GeneratedMethodAccessor9.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl.lambda$invoke$1(AbstractListenerImpl.java:382)
> at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl$$Lambda$388/543782975.run(Unknown Source)
> at org.infinispan.util.concurrent.WithinThreadExecutor.execute(WithinThreadExecutor.java:20)
> at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl.invoke(AbstractListenerImpl.java:404)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl$BaseCacheEntryListenerInvocation.doRealInvocation(CacheNotifierImpl.java:1689)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl$ClusteredListenerInvocation.doRealInvocation(CacheNotifierImpl.java:1586)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl$BaseCacheEntryListenerInvocation.invokeNoChecks(CacheNotifierImpl.java:1680)
> at org.infinispan.notifications.cachelistener.filter.DelegatingCacheEntryListenerInvocation.invokeNoChecks(DelegatingCacheEntryListenerInvocation.java:54)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$Callback.onFilterResult(BaseIckleFilterIndexingServiceProvider.java:261)
> at org.infinispan.objectfilter.impl.predicateindex.MatcherEvalContext.notifyDeltaSubscribers(MatcherEvalContext.java:184)
> at org.infinispan.objectfilter.impl.BaseMatcher.matchDelta(BaseMatcher.java:113)
> at org.infinispan.query.continuous.impl.IckleContinuousQueryFilterIndexingServiceProvider.matchEvent(IckleContinuousQueryFilterIndexingServiceProvider.java:85)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$FilteringListenerInvocation.invoke(BaseIckleFilterIndexingServiceProvider.java:306)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl.notifyCacheEntryModified(CacheNotifierImpl.java:421)
> at org.infinispan.notifications.cachelistener.NotifyHelper.entryCommitted(NotifyHelper.java:67)
> at org.infinispan.interceptors.locking.ClusteringDependentLogic$LocalLogic.commitSingleEntry(ClusteringDependentLogic.java:362)
> at org.infinispan.interceptors.locking.ClusteringDependentLogic$AbstractClusteringDependentLogic.commitEntry(ClusteringDependentLogic.java:190)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitContextEntry(EntryWrappingInterceptor.java:584)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitEntryIfNeeded(EntryWrappingInterceptor.java:813)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitContextEntries(EntryWrappingInterceptor.java:566)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.applyChanges(EntryWrappingInterceptor.java:617)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.applyAndFixVersion(EntryWrappingInterceptor.java:678)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor$$Lambda$379/274655666.accept(Unknown Source)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenAccept(BaseAsyncInterceptor.java:105)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:672)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.visitPutKeyValueCommand(EntryWrappingInterceptor.java:302)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:150)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lambda$nonTxLockAndInvokeNext$1(AbstractLockingInterceptor.java:299)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor$$Lambda$554/57968464.apply(Unknown Source)
> at org.infinispan.interceptors.SyncInvocationStage.addCallback(SyncInvocationStage.java:42)
> at org.infinispan.interceptors.InvocationStage.andHandle(InvocationStage.java:65)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.nonTxLockAndInvokeNext(AbstractLockingInterceptor.java:294)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:126)
> at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitDataWriteCommand(NonTransactionalLockingInterceptor.java:40)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitPutKeyValueCommand(AbstractLockingInterceptor.java:82)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:150)
> at org.infinispan.interceptors.impl.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:217)
> at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:179)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
> at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53)
> at org.infinispan.interceptors.DDAsyncInterceptor.visitPutKeyValueCommand(DDAsyncInterceptor.java:59)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:123)
> at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90)
> at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:234)
> at org.infinispan.cache.impl.CacheImpl.executeCommandAndCommitIfNeededAsync(CacheImpl.java:1930)
> at org.infinispan.cache.impl.CacheImpl.putAsync(CacheImpl.java:1571)
> at org.infinispan.cache.impl.DecoratedCache.putAsync(DecoratedCache.java:690)
> at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.putAsync(AbstractDelegatingAdvancedCache.java:386)
> at org.infinispan.cache.impl.EncoderCache.putAsync(EncoderCache.java:460)
> at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.putAsync(AbstractDelegatingAdvancedCache.java:386)
> at org.infinispan.server.hotrod.CacheRequestProcessor.putInternal(CacheRequestProcessor.java:194)
> at org.infinispan.server.hotrod.CacheRequestProcessor.lambda$put$6(CacheRequestProcessor.java:187)
> at org.infinispan.server.hotrod.CacheRequestProcessor$$Lambda$535/1532854422.run(Unknown Source)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
> at java.lang.Thread.run(Thread.java:748)
> {code}
--
This message was sent by Atlassian Jira
(v7.12.1#712002)
3 years, 5 months
[JBoss JIRA] (ISPN-9867) Client listeners are removed when a notification occurs
by Galder Zamarreño (Jira)
[ https://issues.jboss.org/browse/ISPN-9867?page=com.atlassian.jira.plugin.... ]
Galder Zamarreño updated ISPN-9867:
-----------------------------------
Affects Version/s: 9.4.5.Final
> Client listeners are removed when a notification occurs
> -------------------------------------------------------
>
> Key: ISPN-9867
> URL: https://issues.jboss.org/browse/ISPN-9867
> Project: Infinispan
> Issue Type: Bug
> Components: Hot Rod, Listeners
> Affects Versions: 10.0.0.Alpha2, 9.4.5.Final
> Reporter: William Burns
> Assignee: William Burns
> Priority: Major
> Fix For: 10.0.0.Beta1, 10.0.0.Final
>
>
> Client listeners are removed when a new notification occurs and they find that the channel is closed. This is problematic because this is done while notifying that object in general. This can cause issues if the listener is performing additional operations during this. For example Protobuf checks if an object passes a filter and then notifies it in one call. This will cause deadlocking issues since the filtering requires a read lock and the removal requires a write lock.
> An example of this can be seen in this stack trace:
> {code}
> "HotRod-ServerHandler-7-28" #121 prio=5 os_prio=0 tid=0x0000000003835000 nid=0x51a3 waiting on condition [0x00007f5967d6e000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x00000000e1dd0058> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870)
> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199)
> at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943)
> at org.infinispan.objectfilter.impl.BaseMatcher.unregisterFilter(BaseMatcher.java:228)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$Callback.unregister(BaseIckleFilterIndexingServiceProvider.java:198)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$DelegatingCacheEntryListenerInvocationImpl.unregister(BaseIckleFilterIndexingServiceProvider.java:278)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl.removeListenerInvocation(CacheNotifierImpl.java:1895)
> at org.infinispan.notifications.impl.AbstractListenerImpl.removeListener(AbstractListenerImpl.java:148)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl.removeListener(CacheNotifierImpl.java:1874)
> at org.infinispan.cache.impl.CacheImpl.removeListener(CacheImpl.java:985)
> at org.infinispan.cache.impl.AbstractDelegatingCache.removeListener(AbstractDelegatingCache.java:550)
> at org.infinispan.server.hotrod.ClientListenerRegistry$BaseClientEventSender.isSendEvent(ClientListenerRegistry.java:367)
> at org.infinispan.server.hotrod.ClientListenerRegistry$BaseClientEventSender.onCacheEvent(ClientListenerRegistry.java:350)
> at sun.reflect.GeneratedMethodAccessor9.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl.lambda$invoke$1(AbstractListenerImpl.java:382)
> at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl$$Lambda$388/543782975.run(Unknown Source)
> at org.infinispan.util.concurrent.WithinThreadExecutor.execute(WithinThreadExecutor.java:20)
> at org.infinispan.notifications.impl.AbstractListenerImpl$ListenerInvocationImpl.invoke(AbstractListenerImpl.java:404)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl$BaseCacheEntryListenerInvocation.doRealInvocation(CacheNotifierImpl.java:1689)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl$ClusteredListenerInvocation.doRealInvocation(CacheNotifierImpl.java:1586)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl$BaseCacheEntryListenerInvocation.invokeNoChecks(CacheNotifierImpl.java:1680)
> at org.infinispan.notifications.cachelistener.filter.DelegatingCacheEntryListenerInvocation.invokeNoChecks(DelegatingCacheEntryListenerInvocation.java:54)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$Callback.onFilterResult(BaseIckleFilterIndexingServiceProvider.java:261)
> at org.infinispan.objectfilter.impl.predicateindex.MatcherEvalContext.notifyDeltaSubscribers(MatcherEvalContext.java:184)
> at org.infinispan.objectfilter.impl.BaseMatcher.matchDelta(BaseMatcher.java:113)
> at org.infinispan.query.continuous.impl.IckleContinuousQueryFilterIndexingServiceProvider.matchEvent(IckleContinuousQueryFilterIndexingServiceProvider.java:85)
> at org.infinispan.query.dsl.embedded.impl.BaseIckleFilterIndexingServiceProvider$FilteringListenerInvocation.invoke(BaseIckleFilterIndexingServiceProvider.java:306)
> at org.infinispan.notifications.cachelistener.CacheNotifierImpl.notifyCacheEntryModified(CacheNotifierImpl.java:421)
> at org.infinispan.notifications.cachelistener.NotifyHelper.entryCommitted(NotifyHelper.java:67)
> at org.infinispan.interceptors.locking.ClusteringDependentLogic$LocalLogic.commitSingleEntry(ClusteringDependentLogic.java:362)
> at org.infinispan.interceptors.locking.ClusteringDependentLogic$AbstractClusteringDependentLogic.commitEntry(ClusteringDependentLogic.java:190)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitContextEntry(EntryWrappingInterceptor.java:584)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitEntryIfNeeded(EntryWrappingInterceptor.java:813)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.commitContextEntries(EntryWrappingInterceptor.java:566)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.applyChanges(EntryWrappingInterceptor.java:617)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.applyAndFixVersion(EntryWrappingInterceptor.java:678)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor$$Lambda$379/274655666.accept(Unknown Source)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenAccept(BaseAsyncInterceptor.java:105)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:672)
> at org.infinispan.interceptors.impl.EntryWrappingInterceptor.visitPutKeyValueCommand(EntryWrappingInterceptor.java:302)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:150)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lambda$nonTxLockAndInvokeNext$1(AbstractLockingInterceptor.java:299)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor$$Lambda$554/57968464.apply(Unknown Source)
> at org.infinispan.interceptors.SyncInvocationStage.addCallback(SyncInvocationStage.java:42)
> at org.infinispan.interceptors.InvocationStage.andHandle(InvocationStage.java:65)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.nonTxLockAndInvokeNext(AbstractLockingInterceptor.java:294)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:126)
> at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitDataWriteCommand(NonTransactionalLockingInterceptor.java:40)
> at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitPutKeyValueCommand(AbstractLockingInterceptor.java:82)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:150)
> at org.infinispan.interceptors.impl.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:217)
> at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:179)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
> at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53)
> at org.infinispan.interceptors.DDAsyncInterceptor.visitPutKeyValueCommand(DDAsyncInterceptor.java:59)
> at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:57)
> at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:123)
> at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90)
> at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invokeAsync(AsyncInterceptorChainImpl.java:234)
> at org.infinispan.cache.impl.CacheImpl.executeCommandAndCommitIfNeededAsync(CacheImpl.java:1930)
> at org.infinispan.cache.impl.CacheImpl.putAsync(CacheImpl.java:1571)
> at org.infinispan.cache.impl.DecoratedCache.putAsync(DecoratedCache.java:690)
> at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.putAsync(AbstractDelegatingAdvancedCache.java:386)
> at org.infinispan.cache.impl.EncoderCache.putAsync(EncoderCache.java:460)
> at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.putAsync(AbstractDelegatingAdvancedCache.java:386)
> at org.infinispan.server.hotrod.CacheRequestProcessor.putInternal(CacheRequestProcessor.java:194)
> at org.infinispan.server.hotrod.CacheRequestProcessor.lambda$put$6(CacheRequestProcessor.java:187)
> at org.infinispan.server.hotrod.CacheRequestProcessor$$Lambda$535/1532854422.run(Unknown Source)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
> at java.lang.Thread.run(Thread.java:748)
> {code}
--
This message was sent by Atlassian Jira
(v7.12.1#712002)
3 years, 5 months