[
https://issues.jboss.org/browse/ISPN-6703?page=com.atlassian.jira.plugin....
]
Dan Berindei commented on ISPN-6703:
------------------------------------
The cause is actually
https://github.com/infinispan/infinispan/commit/492ddc640ec22b473e47ca0eb...,
which changed
{{JGroupsTransport.invokeRemotely[Async]}}/{{RpcManager.invokeRemotely[Async]}} to throw
an exception when the response mode is {{SYNCHRONOUS}} and one of the targets returns a
{{CacheNotFoundResponse}}.
The RPC here is an {{IndexUpdateCommand}} that is sent to {{NodeC}} after
{{___defaultcache}} was already stopped there:
{noformat}
15:01:07,301 DEBUG (testng-ClusteredCacheWithInfinispanDirectoryTest:[]) [CacheImpl]
Stopping cache ___defaultcache on NodeC-7028
15:01:07,389 DEBUG (testng-ClusteredCacheWithInfinispanDirectoryTest:[___defaultcache])
[ClusterTopologyManagerImpl] Updating cluster-wide current topology for cache
___defaultcache, topology = CacheTopology{id=3, rebalanceId=1,
currentCH=ReplicatedConsistentHash{ns = 256, owners = (1)[NodeD-18107: 256]},
pendingCH=null, unionCH=null, actualMembers=[NodeD-18107],
persistentUUIDs=[23255a2f-d7e6-476a-bc1f-268e66b45c51]}, availability mode = AVAILABLE
15:01:07,390 TRACE (transport-thread-NodeD-p28-t6:[]) [StateTransferManagerImpl]
Installing new cache topology CacheTopology{id=3, rebalanceId=1,
currentCH=ReplicatedConsistentHash{ns = 256, owners = (1)[NodeD-18107: 256]},
pendingCH=null, unionCH=null, actualMembers=[NodeD-18107],
persistentUUIDs=[23255a2f-d7e6-476a-bc1f-268e66b45c51]} on cache ___defaultcache
15:01:07,411 DEBUG (testng-ClusteredCacheWithInfinispanDirectoryTest:[___defaultcache])
[DirectoryBasedIndexManager] Sending work to backend of type class
org.infinispan.query.indexmanager.InfinispanBackendQueueProcessor
15:01:07,411 DEBUG (testng-ClusteredCacheWithInfinispanDirectoryTest:[___defaultcache])
[RemoteIndexingBackend] ISPN014007: Going to ship list of LuceneWork [PurgeAllLuceneWork:
org.infinispan.query.test.Person] to a remote master indexer
15:01:07,411 TRACE (testng-ClusteredCacheWithInfinispanDirectoryTest:[___defaultcache])
[RpcManagerImpl] NodeD-18107 invoking
IndexUpdateCommand{cacheName='___defaultcache'} to recipient list [NodeC-7028]
with options RpcOptions{timeout=15000, unit=MILLISECONDS, deliverOrder=NONE,
responseFilter=null, responseMode=SYNCHRONOUS, skipReplicationQueue=false}
15:01:07,411 TRACE (OOB-4,NodeD-18107:[]) [JGroupsTransport] Responses:
sender=NodeC-7028value=CacheNotFoundResponse, received=true, suspected=false
15:01:07,412 TRACE (OOB-4,NodeD-18107:[]) [RpcManagerImpl] Replication exception
org.infinispan.remoting.transport.jgroups.SuspectException: Cache not running on node
NodeC-7028
{noformat}
This is the stack trace of the thread invoking {{IndexUpdateCommand}}:
{noformat}
"testng-ClusteredCacheWithInfinispanDirectoryTest@1" prio=5 tid=0x1 nid=NA
waiting
java.lang.Thread.State: WAITING
at sun.misc.Unsafe.park(Unsafe.java:-1)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1695)
at java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323)
at java.util.concurrent.CompletableFuture.timedGet(CompletableFuture.java:1775)
at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1915)
at org.infinispan.util.concurrent.CompletableFutures.await(CompletableFutures.java:75)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)
at
org.infinispan.query.indexmanager.RemoteIndexingBackend.sendCommand(RemoteIndexingBackend.java:136)
at
org.infinispan.query.indexmanager.RemoteIndexingBackend.applyWork(RemoteIndexingBackend.java:72)
at
org.infinispan.query.indexmanager.InfinispanBackendQueueProcessor.applyWork(InfinispanBackendQueueProcessor.java:88)
at
org.hibernate.search.indexes.spi.DirectoryBasedIndexManager.performOperations(DirectoryBasedIndexManager.java:133)
at
org.hibernate.search.backend.impl.WorkQueuePerIndexSplitter.commitOperations(WorkQueuePerIndexSplitter.java:46)
at
org.hibernate.search.backend.impl.BatchedQueueingProcessor.performWorks(BatchedQueueingProcessor.java:81)
at
org.hibernate.search.backend.impl.PerTransactionWorker.performWork(PerTransactionWorker.java:87)
at
org.infinispan.query.backend.QueryInterceptor.performSearchWorks(QueryInterceptor.java:249)
at
org.infinispan.query.backend.QueryInterceptor.purgeAllIndexes(QueryInterceptor.java:217)
at
org.infinispan.query.backend.QueryInterceptor.processClearCommand(QueryInterceptor.java:482)
at
org.infinispan.query.backend.QueryInterceptor.visitClearCommand(QueryInterceptor.java:187)
at
org.infinispan.query.backend.QueryInterceptor.visitClearCommand(QueryInterceptor.java:66)
at org.infinispan.commands.write.ClearCommand.acceptVisitor(ClearCommand.java:44)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.doInvokeNextSync(BaseSequentialInvocationContext.java:265)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.handleActionSync(BaseSequentialInvocationContext.java:287)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.doInvokeNextSync(BaseSequentialInvocationContext.java:272)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.forkInvocationSync(BaseSequentialInvocationContext.java:90)
at
org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:359)
at
org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:290)
at
org.infinispan.statetransfer.StateTransferInterceptor.visitClearCommand(StateTransferInterceptor.java:135)
at
org.infinispan.statetransfer.StateTransferInterceptor.visitClearCommand(StateTransferInterceptor.java:62)
at org.infinispan.commands.write.ClearCommand.acceptVisitor(ClearCommand.java:44)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.doInvokeNextSync(BaseSequentialInvocationContext.java:265)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.handleActionSync(BaseSequentialInvocationContext.java:287)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.doInvokeNextSync(BaseSequentialInvocationContext.java:272)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.forkInvocationSync(BaseSequentialInvocationContext.java:90)
at
org.infinispan.interceptors.impl.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:107)
at
org.infinispan.interceptors.impl.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:76)
at
org.infinispan.interceptors.DDSequentialInterceptor.visitClearCommand(DDSequentialInterceptor.java:82)
at
org.infinispan.interceptors.DDSequentialInterceptor.visitClearCommand(DDSequentialInterceptor.java:47)
at org.infinispan.commands.write.ClearCommand.acceptVisitor(ClearCommand.java:44)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.doInvokeNextSync(BaseSequentialInvocationContext.java:265)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.handleActionSync(BaseSequentialInvocationContext.java:287)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.doInvokeNextSync(BaseSequentialInvocationContext.java:272)
at
org.infinispan.interceptors.impl.BaseSequentialInvocationContext.invokeSync(BaseSequentialInvocationContext.java:253)
at
org.infinispan.interceptors.impl.SequentialInterceptorChainImpl.invoke(SequentialInterceptorChainImpl.java:227)
at org.infinispan.cache.impl.CacheImpl.clear(CacheImpl.java:620)
at org.infinispan.cache.impl.CacheImpl.clear(CacheImpl.java:606)
at org.infinispan.test.TestingUtil.killCaches(TestingUtil.java:770)
at org.infinispan.test.TestingUtil.killCacheManagers(TestingUtil.java:618)
at
org.infinispan.test.MultipleCacheManagersTest.clearContent(MultipleCacheManagersTest.java:101)
{noformat}
Ignoring the {{CacheNotFoundResponse}} was incorrect, but waiting for a new topoloby and
retrying the {{ClearCommand}} isn't correct either. We should catch the
{{SuspectException}} in {{QueryInterceptor}} and retry just the {{IndexUpdateCommand}}, to
account for the fact that the master address is updated some time after the cache topology
was updated.
ClearCommand hanging in the test suite
--------------------------------------
Key: ISPN-6703
URL:
https://issues.jboss.org/browse/ISPN-6703
Project: Infinispan
Issue Type: Bug
Affects Versions: 9.0.0.Alpha2, 8.2.2.Final
Reporter: Gustavo Fernandes
Assignee: Dan Berindei
{noformat}
"testng-ClusteredCacheWithInfinispanDirectoryTest" #15 prio=5 os_prio=0
tid=0x00007fe220532000 nid=0x19a waiting on condition [0x00007fe1cc7dd000]
java.lang.Thread.State: TIMED_WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007b8698d48> (a
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at
org.infinispan.statetransfer.StateTransferLockImpl.waitForTransactionData(StateTransferLockImpl.java:96)
at
org.infinispan.interceptors.base.BaseStateTransferInterceptor.waitForTransactionData(BaseStateTransferInterceptor.java:97)
at
org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:366)
at
org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:281)
at
org.infinispan.statetransfer.StateTransferInterceptor.visitClearCommand(StateTransferInterceptor.java:132)
at org.infinispan.commands.write.ClearCommand.acceptVisitor(ClearCommand.java:44)
at
org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at
org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)
at org.infinispan.commands.AbstractVisitor.visitClearCommand(AbstractVisitor.java:58)
at org.infinispan.commands.write.ClearCommand.acceptVisitor(ClearCommand.java:44)
at
org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at
org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:114)
at
org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:83)
at org.infinispan.commands.AbstractVisitor.visitClearCommand(AbstractVisitor.java:58)
at org.infinispan.commands.write.ClearCommand.acceptVisitor(ClearCommand.java:44)
at
org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)
at
org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)
at org.infinispan.commands.AbstractVisitor.visitClearCommand(AbstractVisitor.java:58)
at org.infinispan.commands.write.ClearCommand.acceptVisitor(ClearCommand.java:44)
at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:335)
at org.infinispan.cache.impl.CacheImpl.clear(CacheImpl.java:588)
at org.infinispan.cache.impl.CacheImpl.clear(CacheImpl.java:579)
at org.infinispan.test.TestingUtil.killCaches(TestingUtil.java:769)
at org.infinispan.test.TestingUtil.killCacheManagers(TestingUtil.java:607)
at
org.infinispan.test.MultipleCacheManagersTest.clearContent(MultipleCacheManagersTest.java:101)
at sun.reflect.GeneratedMethodAccessor214.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84)
at org.testng.internal.Invoker.invokeConfigurationMethod(Invoker.java:564)
at org.testng.internal.Invoker.invokeConfigurations(Invoker.java:213)
at org.testng.internal.Invoker.invokeMethod(Invoker.java:786)
at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901)
at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231)
at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127)
at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111)
at org.testng.TestRunner.privateRun(TestRunner.java:767)
at org.testng.TestRunner.run(TestRunner.java:617)
at org.testng.SuiteRunner.runTest(SuiteRunner.java:348)
at org.testng.SuiteRunner.access$000(SuiteRunner.java:38)
at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:382)
at org.testng.internal.thread.ThreadUtil$2.call(ThreadUtil.java:64)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
{noformat}
--
This message was sent by Atlassian JIRA
(v6.4.11#64026)