ReplicationQueue ISPN000072: Failed replicating XXX elements in replication queue org.infinispan.commons.CacheException: java.lang.RuntimeException: Failure to marshal argument(s ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Key: ISPN-9300 URL: https://issues.jboss.org/browse/ISPN-9300 Project: Infinispan Issue Type: Bug Components: Core Affects Versions: 7.2.4.Final Reporter: Nilesh Deshpande *I am getting continuous error in the log for following.* ERROR 06/16 21:38:03.910 [replicationQueue-thread--p4-t1] o.i.r.ReplicationQueue ISPN000072: Failed replicating 114 elements in replication queue org.infinispan.commons.CacheException: java.lang.RuntimeException: Failure to marshal argument(s) at org.infinispan.commons.util.Util.rewrapAsCacheException(Util.java:657) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:250) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:561) at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:287) at org.infinispan.remoting.ReplicationQueueImpl.flush(ReplicationQueueImpl.java:152) at org.infinispan.remoting.ReplicationQueueImpl$1.run(ReplicationQueueImpl.java:103) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: Failure to marshal argument(s) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:417) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:432) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:241) ... 11 common frames omitted Caused by: java.util.ConcurrentModificationException: null at java.util.HashMap$HashIterator.nextNode(HashMap.java:1442) at java.util.HashMap$KeyIterator.next(HashMap.java:1466) at org.infinispan.commons.marshall.MarshallUtil.marshallCollection(MarshallUtil.java:24) at org.infinispan.marshall.exts.SetExternalizer.writeObject(SetExternalizer.java:44) at org.infinispan.marshall.exts.SetExternalizer.writeObject(SetExternalizer.java:26) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:442) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:144) at org.jboss.marshalling.river.RiverMarshaller.doWriteFields(RiverMarshaller.java:1062) at org.jboss.marshalling.river.RiverMarshaller.doWriteSerializableObject(RiverMarshaller.java:1018) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:884) at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.writeCommandParameters(ReplicableCommandExternalizer.java:58) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.writeObject(ReplicableCommandExternalizer.java:43) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.writeObject(ReplicableCommandExternalizer.java:31) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:442) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:144) at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.writeCommandParameters(ReplicableCommandExternalizer.java:58) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.marshallParameters(CacheRpcCommandExternalizer.java:124) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.writeObject(CacheRpcCommandExternalizer.java:108) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.writeObject(CacheRpcCommandExternalizer.java:65) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:442) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:144) at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) at org.infinispan.marshall.exts.ReplicableCommandExternalizer.writeCommandParameters(ReplicableCommandExternalizer.java:58) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.marshallParameters(CacheRpcCommandExternalizer.java:124) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.writeObject(CacheRpcCommandExternalizer.java:108) at org.infinispan.marshall.exts.CacheRpcCommandExternalizer.writeObject(CacheRpcCommandExternalizer.java:65) at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.writeObject(ExternalizerTable.java:442) at org.jboss.marshalling.river.RiverMarshaller.doWriteObject(RiverMarshaller.java:144) at org.jboss.marshalling.AbstractObjectOutput.writeObject(AbstractObjectOutput.java:58) at org.jboss.marshalling.AbstractMarshaller.writeObject(AbstractMarshaller.java:111) at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.objectToObjectStream(AbstractJBossMarshaller.java:72) at org.infinispan.marshall.core.VersionAwareMarshaller.objectToBuffer(VersionAwareMarshaller.java:77) at org.infinispan.commons.marshall.AbstractMarshaller.objectToBuffer(AbstractMarshaller.java:41) at org.infinispan.commons.marshall.AbstractDelegatingMarshaller.objectToBuffer(AbstractDelegatingMarshaller.java:85) at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectToBuffer(MarshallerAdapter.java:23) at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:415) ... 13 common frames omitted Caused by: org.jboss.marshalling.TraceInformation: null *My cache setting is following* <?xml version="1.0" encoding="UTF-8"?> <infinispan xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:infinispan:config:7.0 http://www.infinispan.org/schemas/infinispan-config-7.0.xsd" xmlns="urn:infinispan:config:7.0"> <jgroups> <stack-file name="configurationFile" path="cache-replicator.conf"/> </jgroups> <cache-container default-cache="defaultCache"> <!-- Enables collection of global JMX statistics, and allows you to customize the JMX domain name under which MBeans are registered. --> <jmx> <property name="enabled">false</property> <property name="jmxDomain">org.infinispan</property> <property name="cacheManagerName">CacheManager</property> <property name="statistics">false</property> </jmx> <!-- If the transport is omitted, there is no way to create distributed or clustered caches. There is no added cost to defining a transport and not creating a cache that uses one, since the transport is created and initialized lazily. By default, the JGroupsTransport is used. Note that the JGroups transport uses sensible defaults if no configuration property is defined. See: http://community.jboss.org/wiki/ClusteredConfigurationQuickStart http://docs.jboss.org/infinispan/5.0/apidocs/org/infinispan/remoting/tran... --> <transport cluster="TrueSight-Cache-cluster" stack="configurationFile"> </transport> <!-- Used to register JVM shutdown hooks. hookBehavior: DEFAULT, REGISTER, DONT_REGISTER --> <!-- <shutdown hookBehavior="DONT_REGISTER"/> --> <local-cache name="defaultCache" statistics="false"> <!-- <storeAsBinary /> --> <!-- Tunes the locking and concurrency characteristics of the cache. See: http://community.jboss.org/wiki/LockingandConcurrency --> <locking isolation="READ_COMMITTED" acquire-timeout="20000" write-skew="false" concurrency-level="5000" striping="false" /> <!-- Used to register JMX statistics in any available MBean server <jmxStatistics enabled="false"/>--> <!-- Used to control whether objects stored in Infinispan nodes are primarily stored as rehydrated Objects or as raw byte arrays. See: http://community.jboss.org/wiki/Marshalling#Lazy_Deserialization --> <!-- <storeAsBinary enabled="true"/> --> <!-- Used to enable invocation batching and allow the use of Cache.startBatch()/endBatch() methods. http://community.jboss.org/wiki/Batching <invocationBatching enabled="true"/> --> <transaction mode="BATCH"/> <!-- This element specifies that the cache is clustered. modes supported: distribution (d), replication (r) or invalidation (i). http://community.jboss.org/wiki/Clusteringmodes --> </local-cache> <replicated-cache name="defaultCache" mode="ASYNC" queue-size="500" queue-flush-interval="10000"> <state-transfer await-initial-transfer="false" timeout="240000" enabled="true" chunk-size="10000" /> <!--Default write-skew is false, concurrency-level is 16 and acquire-timeout is 20 seconds in infinispan. Increasing --> <locking concurrency-level="1000" striping="true" acquire-timeout="240000" isolation="READ_COMMITTED" write-skew="false"/> <!-- <sync replTimeout="20000"/> --> <!-- async replication. --> </replicated-cache> </cache-container> </infinispan>