[ https://issues.jboss.org/browse/ISPN-7779?page=com.atlassian.jira.plugin.... ] AHMAD ELABIARY updated ISPN-7779: --------------------------------- Description: I am currently testing remote deployment of JBoss Data Grid 7.1 with a 3-node replicated cluster. Items stored in the replicated cache are protobuf encoded and indexed. I have enabled global-state persistence on each of the cluster nodes to ensure that the protobuf schema cache is not purged on startup. However, when state-transfer occurs to a node joining the cluster, the transfer triggers an array of exceptions as shown below. If i then kill the nodes transferring state to this erroring new joiner, the new joiner node continues to start-up and only at that point is this log message show {code:java} 15:54:19,991 INFO [org.jboss.as.clustering.infinispan] (MSC service thread 1-4) DGISPN0001: Started ___protobuf_metadata cache from clustered container {code} It seems as through the protobuf metadata cache starts after the configured caches which means that during their startup they do not have access to the persisted schema *State Transfer Exception* {code:java} 2017-04-28 15:03:08,036 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (stateTransferExecutor-thread--p7-t14) ISPN000136: Error executing command PutKeyValueCommand, writing keys [[B0x4A0E4C45494C4757..[16]]: org.hibernate.search.bridge.BridgeException: Exception while calling bridge#set entity class: org.infinispan.query.remote.impl.indexing.ProtobufValueWrapper field bridge: org.infinispan.query.remote.impl.indexing.ProtobufValueWrapperFieldBridge@44f36069 at org.hibernate.search.bridge.util.impl.ContextualExceptionBridgeHelper.buildBridgeException(ContextualExceptionBridgeHelper.java:103) at org.hibernate.search.bridge.util.impl.ContextualExceptionBridgeHelper$OneWayConversionContextImpl.set(ContextualExceptionBridgeHelper.java:137) at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.buildDocumentFieldForClassBridges(DocumentBuilderIndexedEntity.java:890) at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.buildDocumentFields(DocumentBuilderIndexedEntity.java:454) at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.getDocument(DocumentBuilderIndexedEntity.java:391) at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.createUpdateWork(DocumentBuilderIndexedEntity.java:301) at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.addWorkToQueue(DocumentBuilderIndexedEntity.java:243) at org.hibernate.search.engine.impl.WorkPlan$PerEntityWork.enqueueLuceneWork(WorkPlan.java:520) at org.hibernate.search.engine.impl.WorkPlan$PerClassWork.enqueueLuceneWork(WorkPlan.java:282) at org.hibernate.search.engine.impl.WorkPlan.getPlannedLuceneWork(WorkPlan.java:154) at org.hibernate.search.backend.impl.WorkQueue.prepareWorkPlan(WorkQueue.java:114) at org.hibernate.search.backend.impl.BatchedQueueingProcessor.prepareWorks(BatchedQueueingProcessor.java:57) at org.hibernate.search.backend.impl.PerTransactionWorker.performWork(PerTransactionWorker.java:86) at org.infinispan.query.backend.QueryInterceptor.performSearchWorks(QueryInterceptor.java:241) at org.infinispan.query.backend.QueryInterceptor.performSearchWork(QueryInterceptor.java:235) at org.infinispan.query.backend.QueryInterceptor.updateIndexes(QueryInterceptor.java:228) at org.infinispan.query.backend.QueryInterceptor.processPutKeyValueCommand(QueryInterceptor.java:457) at org.infinispan.query.backend.QueryInterceptor.lambda$visitPutKeyValueCommand$0(QueryInterceptor.java:154) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenAccept(BaseAsyncInterceptor.java:108) at org.infinispan.query.backend.QueryInterceptor.visitPutKeyValueCommand(QueryInterceptor.java:154) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:57) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:107) at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitDataWriteCommand(NonTransactionalLockingInterceptor.java:39) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitPutKeyValueCommand(AbstractLockingInterceptor.java:77) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:57) at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:397) at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:340) at org.infinispan.statetransfer.StateTransferInterceptor.visitPutKeyValueCommand(StateTransferInterceptor.java:123) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:57) at org.infinispan.interceptors.impl.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:164) at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:153) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:57) at org.infinispan.interceptors.impl.IsMarshallableInterceptor.visitPutKeyValueCommand(IsMarshallableInterceptor.java:49) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenApply(BaseAsyncInterceptor.java:77) at org.infinispan.interceptors.compat.BaseTypeConverterInterceptor.visitPutKeyValueCommand(BaseTypeConverterInterceptor.java:82) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:126) at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:92) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53) at org.infinispan.interceptors.DDAsyncInterceptor.visitPutKeyValueCommand(DDAsyncInterceptor.java:59) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:49) at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:282) at org.infinispan.statetransfer.StateConsumerImpl.doApplyState(StateConsumerImpl.java:621) at org.infinispan.statetransfer.StateConsumerImpl.applyChunk(StateConsumerImpl.java:589) at org.infinispan.statetransfer.StateConsumerImpl.access$000(StateConsumerImpl.java:93) at org.infinispan.statetransfer.StateConsumerImpl$1.run(StateConsumerImpl.java:552) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.IllegalArgumentException: Message descriptor not found : "******** I HAVE MASKED THIS VALUE *********" at org.infinispan.protostream.impl.SerializationContextImpl.getMessageDescriptor(SerializationContextImpl.java:166) at org.infinispan.query.remote.impl.indexing.WrappedMessageTagHandler.onTag(WrappedMessageTagHandler.java:65) at org.infinispan.protostream.ProtobufParser.parseMessage(ProtobufParser.java:75) at org.infinispan.protostream.ProtobufParser.parseInternal(ProtobufParser.java:57) at org.infinispan.protostream.ProtobufParser.parse(ProtobufParser.java:44) at org.infinispan.query.remote.impl.indexing.ProtobufValueWrapperFieldBridge.decodeAndIndex(ProtobufValueWrapperFieldBridge.java:58) at org.infinispan.query.remote.impl.indexing.ProtobufValueWrapperFieldBridge.set(ProtobufValueWrapperFieldBridge.java:46) at org.hibernate.search.bridge.util.impl.ContextualExceptionBridgeHelper$OneWayConversionContextImpl.set(ContextualExceptionBridgeHelper.java:134) ... 56 more {code} was: I am currently testing remote deployment of JBoss Data Grid 7.1 with a 3-node replicated cluster. Items stored in the replicated cache are protobuf encoded and indexed. I have enabled global-state persistence on each of the cluster nodes to ensure that the protobuf schema cache is not purged on startup. However, when state-transfer occurs to a node joining the cluster, the transfer triggers an array of exceptions as shown below. If i then kill the nodes transferring state to this erroring new joiner, the new joiner node continues to start-up and only at that point is this log message show {code:java} 15:54:19,991 INFO [org.jboss.as.clustering.infinispan] (MSC service thread 1-4) DGISPN0001: Started ___protobuf_metadata cache from clustered container {code} It seems as through the protobuf metadata cache starts after the configured caches which means that during their startup they do not have access to the persisted schema *State Transfer Exception* {code:java} 2017-04-28 15:03:08,036 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (stateTransferExecutor-thread--p7-t14) ISPN000136: Error executing command PutKeyValueCommand, writing keys [[B0x4A0E4C45494C4757..[16]]: org.hibernate.search.bridge.BridgeException: Exception while calling bridge#set entity class: org.infinispan.query.remote.impl.indexing.ProtobufValueWrapper field bridge: org.infinispan.query.remote.impl.indexing.ProtobufValueWrapperFieldBridge@44f36069 at org.hibernate.search.bridge.util.impl.ContextualExceptionBridgeHelper.buildBridgeException(ContextualExceptionBridgeHelper.java:103) at org.hibernate.search.bridge.util.impl.ContextualExceptionBridgeHelper$OneWayConversionContextImpl.set(ContextualExceptionBridgeHelper.java:137) at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.buildDocumentFieldForClassBridges(DocumentBuilderIndexedEntity.java:890) at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.buildDocumentFields(DocumentBuilderIndexedEntity.java:454) at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.getDocument(DocumentBuilderIndexedEntity.java:391) at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.createUpdateWork(DocumentBuilderIndexedEntity.java:301) at org.hibernate.search.engine.spi.DocumentBuilderIndexedEntity.addWorkToQueue(DocumentBuilderIndexedEntity.java:243) at org.hibernate.search.engine.impl.WorkPlan$PerEntityWork.enqueueLuceneWork(WorkPlan.java:520) at org.hibernate.search.engine.impl.WorkPlan$PerClassWork.enqueueLuceneWork(WorkPlan.java:282) at org.hibernate.search.engine.impl.WorkPlan.getPlannedLuceneWork(WorkPlan.java:154) at org.hibernate.search.backend.impl.WorkQueue.prepareWorkPlan(WorkQueue.java:114) at org.hibernate.search.backend.impl.BatchedQueueingProcessor.prepareWorks(BatchedQueueingProcessor.java:57) at org.hibernate.search.backend.impl.PerTransactionWorker.performWork(PerTransactionWorker.java:86) at org.infinispan.query.backend.QueryInterceptor.performSearchWorks(QueryInterceptor.java:241) at org.infinispan.query.backend.QueryInterceptor.performSearchWork(QueryInterceptor.java:235) at org.infinispan.query.backend.QueryInterceptor.updateIndexes(QueryInterceptor.java:228) at org.infinispan.query.backend.QueryInterceptor.processPutKeyValueCommand(QueryInterceptor.java:457) at org.infinispan.query.backend.QueryInterceptor.lambda$visitPutKeyValueCommand$0(QueryInterceptor.java:154) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenAccept(BaseAsyncInterceptor.java:108) at org.infinispan.query.backend.QueryInterceptor.visitPutKeyValueCommand(QueryInterceptor.java:154) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:57) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:107) at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitDataWriteCommand(NonTransactionalLockingInterceptor.java:39) at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitPutKeyValueCommand(AbstractLockingInterceptor.java:77) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:57) at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:397) at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:340) at org.infinispan.statetransfer.StateTransferInterceptor.visitPutKeyValueCommand(StateTransferInterceptor.java:123) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:57) at org.infinispan.interceptors.impl.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:164) at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:153) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:57) at org.infinispan.interceptors.impl.IsMarshallableInterceptor.visitPutKeyValueCommand(IsMarshallableInterceptor.java:49) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextThenApply(BaseAsyncInterceptor.java:77) at org.infinispan.interceptors.compat.BaseTypeConverterInterceptor.visitPutKeyValueCommand(BaseTypeConverterInterceptor.java:82) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:126) at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:92) at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53) at org.infinispan.interceptors.DDAsyncInterceptor.visitPutKeyValueCommand(DDAsyncInterceptor.java:59) at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:49) at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:282) at org.infinispan.statetransfer.StateConsumerImpl.doApplyState(StateConsumerImpl.java:621) at org.infinispan.statetransfer.StateConsumerImpl.applyChunk(StateConsumerImpl.java:589) at org.infinispan.statetransfer.StateConsumerImpl.access$000(StateConsumerImpl.java:93) at org.infinispan.statetransfer.StateConsumerImpl$1.run(StateConsumerImpl.java:552) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.IllegalArgumentException: Message descriptor not found : flightpricestore.SectorDayCacheEntry at org.infinispan.protostream.impl.SerializationContextImpl.getMessageDescriptor(SerializationContextImpl.java:166) at org.infinispan.query.remote.impl.indexing.WrappedMessageTagHandler.onTag(WrappedMessageTagHandler.java:65) at org.infinispan.protostream.ProtobufParser.parseMessage(ProtobufParser.java:75) at org.infinispan.protostream.ProtobufParser.parseInternal(ProtobufParser.java:57) at org.infinispan.protostream.ProtobufParser.parse(ProtobufParser.java:44) at org.infinispan.query.remote.impl.indexing.ProtobufValueWrapperFieldBridge.decodeAndIndex(ProtobufValueWrapperFieldBridge.java:58) at org.infinispan.query.remote.impl.indexing.ProtobufValueWrapperFieldBridge.set(ProtobufValueWrapperFieldBridge.java:46) at org.hibernate.search.bridge.util.impl.ContextualExceptionBridgeHelper$OneWayConversionContextImpl.set(ContextualExceptionBridgeHelper.java:134) ... 56 more {code} -- This message was sent by Atlassian JIRA (v7.2.3#72005)