JBoss Community

I have two nodes, A and B. A is up and running and there data in the cache.  When I start node B, the state transfer fails with the following exception:

2010-11-17 14:56:11,499 ERROR [org.jboss.kernel.plugins.dependency.AbstractKernelController] (main) Error installing to Start: name=EcommerceCoreCache state=Create
2010-11-17 14:56:11,499 ERROR [org.jboss.kernel.plugins.dependency.AbstractKernelController] (main) Error installing to Start: name=EcommerceCoreCache state=Create
org.jboss.cache.CacheException: java.lang.reflect.InvocationTargetException
     at org.jboss.cache.util.reflect.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:148)
     at org.jboss.cache.factories.ComponentRegistry$PrioritizedMethod.invoke(ComponentRegistry.java:1012)
     at org.jboss.cache.factories.ComponentRegistry.internalStart(ComponentRegistry.java:778)
     at org.jboss.cache.factories.ComponentRegistry.start(ComponentRegistry.java:632)
     at org.jboss.cache.invocation.CacheInvocationDelegate.start(CacheInvocationDelegate.java:345)
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
     at java.lang.reflect.Method.invoke(Method.java:597)
     at org.jboss.reflect.plugins.introspection.ReflectionUtils.invoke(ReflectionUtils.java:59)
     at org.jboss.reflect.plugins.introspection.ReflectMethodInfoImpl.invoke(ReflectMethodInfoImpl.java:150)
     at org.jboss.joinpoint.plugins.BasicMethodJoinPoint.dispatch(BasicMethodJoinPoint.java:66)
     at org.jboss.kernel.plugins.dependency.KernelControllerContextAction$JoinpointDispatchWrapper.execute(KernelControllerContextAction.java:241)
     at org.jboss.kernel.plugins.dependency.ExecutionWrapper.execute(ExecutionWrapper.java:47)
     at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchExecutionWrapper(KernelControllerContextAction.java:109)
     at org.jboss.kernel.plugins.dependency.KernelControllerContextAction.dispatchJoinPoint(KernelControllerContextAction.java:70)
     at org.jboss.kernel.plugins.dependency.LifecycleAction.installActionInternal(LifecycleAction.java:221)
     at org.jboss.kernel.plugins.dependency.InstallsAwareAction.installAction(InstallsAwareAction.java:54)
     at org.jboss.kernel.plugins.dependency.InstallsAwareAction.installAction(InstallsAwareAction.java:42)
     at org.jboss.dependency.plugins.action.SimpleControllerContextAction.simpleInstallAction(SimpleControllerContextAction.java:62)
     at org.jboss.dependency.plugins.action.AccessControllerContextAction.install(AccessControllerContextAction.java:71)
     at org.jboss.dependency.plugins.AbstractControllerContextActions.install(AbstractControllerContextActions.java:51)
     at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:348)
     at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1631)
     at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:934)
     at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:1082)
     at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:984)
     at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:774)
     at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:540)
     at org.jboss.deployers.vfs.deployer.kernel.BeanMetaDataDeployer.deploy(BeanMetaDataDeployer.java:121)
     at org.jboss.deployers.vfs.deployer.kernel.BeanMetaDataDeployer.deploy(BeanMetaDataDeployer.java:51)
     at org.jboss.deployers.spi.deployer.helpers.AbstractSimpleRealDeployer.internalDeploy(AbstractSimpleRealDeployer.java:62)
     at org.jboss.deployers.spi.deployer.helpers.AbstractRealDeployer.deploy(AbstractRealDeployer.java:50)
     at org.jboss.deployers.plugins.deployers.DeployerWrapper.deploy(DeployerWrapper.java:171)
     at org.jboss.deployers.plugins.deployers.DeployersImpl.doDeploy(DeployersImpl.java:1439)
     at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:1157)
     at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:1178)
     at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:1210)
     at org.jboss.deployers.plugins.deployers.DeployersImpl.install(DeployersImpl.java:1098)
     at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:348)
     at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1631)
     at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:934)
     at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:1082)
     at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:984)
     at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:822)
     at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:553)
     at org.jboss.deployers.plugins.deployers.DeployersImpl.process(DeployersImpl.java:781)
     at org.jboss.deployers.plugins.main.MainDeployerImpl.process(MainDeployerImpl.java:702)
     at org.jboss.system.server.profileservice.repository.MainDeployerAdapter.process(MainDeployerAdapter.java:117)
     at org.jboss.system.server.profileservice.repository.ProfileDeployAction.install(ProfileDeployAction.java:70)
     at org.jboss.system.server.profileservice.repository.AbstractProfileAction.install(AbstractProfileAction.java:53)
     at org.jboss.system.server.profileservice.repository.AbstractProfileService.install(AbstractProfileService.java:361)
     at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:348)
     at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1631)
     at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:934)
     at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:1082)
     at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:984)
     at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:822)
     at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:553)
     at org.jboss.system.server.profileservice.repository.AbstractProfileService.activateProfile(AbstractProfileService.java:306)
     at org.jboss.system.server.profileservice.ProfileServiceBootstrap.start(ProfileServiceBootstrap.java:271)
     at org.jboss.bootstrap.AbstractServerImpl.start(AbstractServerImpl.java:461)
     at org.jboss.Main.boot(Main.java:221)
     at org.jboss.Main$1.run(Main.java:556)
     at java.lang.Thread.run(Thread.java:619)
Caused by: java.lang.reflect.InvocationTargetException
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
     at java.lang.reflect.Method.invoke(Method.java:597)
     at org.jboss.cache.util.reflect.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:144)
     ... 64 more
Caused by: org.jboss.cache.CacheException: Unable to fetch state on startup
     at org.jboss.cache.RPCManagerImpl.start(RPCManagerImpl.java:396)
     ... 69 more
Caused by: org.jboss.cache.CacheException: java.io.EOFException
     at org.jboss.cache.statetransfer.DefaultStateTransferIntegrator.integrateTransientState(DefaultStateTransferIntegrator.java:251)
     at org.jboss.cache.statetransfer.DefaultStateTransferIntegrator.integrateState(DefaultStateTransferIntegrator.java:113)
     at org.jboss.cache.statetransfer.DefaultStateTransferManager.setState(DefaultStateTransferManager.java:199)
     at org.jboss.cache.statetransfer.DefaultStateTransferManager.setState(DefaultStateTransferManager.java:163)
     at org.jboss.cache.remoting.jgroups.ChannelMessageListener.setState(ChannelMessageListener.java:296)
     at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.handleUpEvent(MessageDispatcher.java:714)
     at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:776)
     at org.jgroups.JChannel.up(JChannel.java:1291)
     at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:462)
     at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:501)
     at org.jgroups.protocols.pbcast.STREAMING_STATE_TRANSFER.connectToStateProvider(STREAMING_STATE_TRANSFER.java:566)
     at org.jgroups.protocols.pbcast.STREAMING_STATE_TRANSFER.handleStateRsp(STREAMING_STATE_TRANSFER.java:503)
     at org.jgroups.protocols.pbcast.STREAMING_STATE_TRANSFER.up(STREAMING_STATE_TRANSFER.java:271)
     at org.jgroups.protocols.FRAG2.up(FRAG2.java:188)
     at org.jgroups.protocols.FC.up(FC.java:473)
     at org.jgroups.protocols.pbcast.GMS.up(GMS.java:824)
     at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:233)
     at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:616)
     at org.jgroups.protocols.UNICAST.up(UNICAST.java:282)
     at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:758)
     at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167)
     at org.jgroups.protocols.FD.up(FD.java:284)
     at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:309)
     at org.jgroups.protocols.MERGE2.up(MERGE2.java:144)
     at org.jgroups.protocols.Discovery.up(Discovery.java:264)
     at org.jgroups.protocols.PING.up(PING.java:273)
     at org.jgroups.protocols.TP.passMessageUp(TP.java:1285)
     at org.jgroups.protocols.TP.access$100(TP.java:49)
     at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1838)
     at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1812)
     at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
     ... 1 more
Caused by: java.io.EOFException
     at java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2281)
     at java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:2750)
     at java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:780)
     at java.io.ObjectInputStream.<init>(ObjectInputStream.java:280)
     at org.drools.common.DroolsObjectInputStream.<init>(DroolsObjectInputStream.java:55)
     at org.drools.common.DroolsObjectInputStream.<init>(DroolsObjectInputStream.java:49)
     at org.drools.common.AbstractRuleBase.readExternal(AbstractRuleBase.java:232)
     at org.drools.reteoo.ReteooRuleBase.readExternal(ReteooRuleBase.java:179)
     at org.drools.impl.KnowledgeBaseImpl.readExternal(KnowledgeBaseImpl.java:87)
     at java.io.ObjectInputStream.readExternalData(ObjectInputStream.java:1792)
     at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1751)
     at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
     at java.io.ObjectInputStream.readObject(ObjectInputStream.java:351)
     at java.util.HashMap.readObject(HashMap.java:1030)
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
     at java.lang.reflect.Method.invoke(Method.java:597)
     at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:974)
     at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1849)
     at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753)
     at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
     at java.io.ObjectInputStream.readObject(ObjectInputStream.java:351)
     at org.jboss.cache.marshall.NodeData.readExternal(NodeData.java:112)
     at org.jboss.cache.marshall.CacheMarshaller200.unmarshallObject(CacheMarshaller200.java:685)
     at org.jboss.cache.marshall.CacheMarshaller200.populateFromStream(CacheMarshaller200.java:875)
     at org.jboss.cache.marshall.CacheMarshaller200.unmarshallLinkedList(CacheMarshaller200.java:818)
     at org.jboss.cache.marshall.CacheMarshaller200.unmarshallObject(CacheMarshaller200.java:649)
     at org.jboss.cache.marshall.CacheMarshaller200.unmarshallObject(CacheMarshaller200.java:589)
     at org.jboss.cache.marshall.CacheMarshaller200.objectFromObjectStream(CacheMarshaller200.java:161)
     at org.jboss.cache.marshall.VersionAwareMarshaller.objectFromObjectStream(VersionAwareMarshaller.java:360)
     at org.jboss.cache.statetransfer.DefaultStateTransferIntegrator.readNodesAsList(DefaultStateTransferIntegrator.java:405)
     at org.jboss.cache.statetransfer.DefaultStateTransferIntegrator.integrateTransientState(DefaultStateTransferIntegrator.java:362)
     at org.jboss.cache.statetransfer.DefaultStateTransferIntegrator.integrateTransientState(DefaultStateTransferIntegrator.java:234)
     ... 32 more

There only one node in the cache and I don't think it is too big.  The exception is thrown almost immediately so I do not think it is a timeout problem.   My cache configuration is this:

 <bean name="EcommerceCoreCacheConfig" class="org.jboss.cache.config.Configuration">



  <property name="nodeLockingScheme">MVCC</property>

  <property name="useLockStriping">false</property>

  <property name="cacheMode">REPL_SYNC</property> 

  <property name="isolationLevel">READ_COMMITTED</property>

  <property name="clusterName">${jboss.partition.name:DefaultPartition}-ecommerce-core-cache</property>     

  <property name="multiplexerStack">${jboss.default.jgroups.stack:udp}</property>



  <!-- Whether or not to fetch state on joining a cluster. -->

  <property name="fetchInMemoryState">true</property>



     <!-- The max amount of time (in milliseconds) we wait until the

       state (ie. the contents of the cache) are retrieved from

       existing members at startup. Ignored if FetchInMemoryState=false. 



           5 minutes to get the content when joining a cluster.  

     -->

     <property name="stateRetrievalTimeout">300000</property>



     <!-- Number of milliseconds to wait until all responses for a

           synchronous call have been received. 

     -->

     <property name="syncReplTimeout">17500</property>



     <!-- Max number of milliseconds to wait for a lock acquisition -->

     <property name="lockAcquisitionTimeout">15000</property>

     <property name="useRegionBasedMarshalling">false</property>

     <property name="inactiveOnStartup">false</property>

     <property name="serializationExecutorPoolSize">0</property>        

     <property name="listenerAsyncPoolSize">0</property>         

     <property name="evictionConfig">

     <bean class="org.jboss.cache.config.EvictionConfig">

     <!-- evict every one minute -->

          <property name="wakeupInterval">60000</property>

          <!-- Overall default -->

          <property name="defaultEvictionRegionConfig">

               <bean class="org.jboss.cache.config.EvictionRegionConfig">

                    <property name="regionName">/</property>

                    <property name="evictionAlgorithmConfig">

                    <bean class="org.jboss.cache.eviction.LRUAlgorithmConfig">

                                   <property name="maxNodes">100</property>
                            <property name="maxAge">300000</property>
                                    <property name="minTimeToLiveSeconds">60000</property>
                    </bean>
                    </property>

               </bean>

          </property>

          <property name="evictionRegionConfigs">
          <list>

          <bean class="org.jboss.cache.config.EvictionRegionConfig">

          <property name="regionName">/ecommerce/rules</property>

          <property name="evictionAlgorithmConfig">

          <bean class="org.jboss.cache.eviction.LRUAlgorithmConfig">

          <property name="maxNodes">20</property>

          <property name="timeToLiveSeconds">60000</property>
          <property name="maxAge">86400000</property>
          </bean>

          </property>

          </bean>

          </list>

     </property>

</bean>

</property>

</bean>

        

This is running on JBoss 5.1.0.GA servers and UDP configuration is the default JBoss UPD protocol stack.  Reproduced here:

<!ENTITY shared-udp '
          <!--  UDP transport config meant to be shared between different channels
                with different requirements. Message bundling is disabled in this 
                general-purpose config as it can add latency to synchronous RPCs. -->
          <UDP
               singleton_name="shared-udp"
               mcast_port="${jboss.jgroups.udp.mcast_port:45688}"
               mcast_addr="${jboss.partition.udpGroup:228.11.11.11}"
               tos="8"
               ucast_recv_buf_size="20000000"
               ucast_send_buf_size="640000"
               mcast_recv_buf_size="25000000"
               mcast_send_buf_size="640000"
               loopback="true"
               discard_incompatible_packets="true"
               enable_bundling="false"
               max_bundle_size="64000"
               max_bundle_timeout="30"
               use_incoming_packet_handler="true"
               ip_ttl="${jgroups.udp.ip_ttl:2}"
               thread_naming_pattern="cl"
               timer.num_threads="12"
           enable_diagnostics="${jboss.jgroups.enable_diagnostics:true}"
           diagnostics_addr="${jboss.jgroups.diagnostics_addr:224.0.0.75}"
           diagnostics_port="${jboss.jgroups.diagnostics_port:7500}"
               use_concurrent_stack="true"
                  thread_pool.enabled="true"
               thread_pool.min_threads="20"
               thread_pool.max_threads="200"
               thread_pool.keep_alive_time="5000"
               thread_pool.queue_enabled="true"
               thread_pool.queue_max_size="1000"
               thread_pool.rejection_policy="discard"
               oob_thread_pool.enabled="true"
               oob_thread_pool.min_threads="1"
               oob_thread_pool.max_threads="20"
               oob_thread_pool.keep_alive_time="5000"
               oob_thread_pool.queue_enabled="false"
               oob_thread_pool.queue_max_size="100"
               oob_thread_pool.rejection_policy="run"/>
          '>
]>
....
    <stack name="udp"
           description="Default: IP multicast based stack, with flow control.">
        <config>
          &shared-udp;
          <PING timeout="2000" num_initial_members="3"/>
          <MERGE2 max_interval="100000" min_interval="20000"/>
          <FD_SOCK/>
          <FD timeout="6000" max_tries="5" shun="true"/>
          <VERIFY_SUSPECT timeout="1500"/>
          <pbcast.NAKACK use_mcast_xmit="false" gc_lag="0"
                   retransmit_timeout="300,600,1200,2400,4800"
                   discard_delivered_msgs="true"/>
          <UNICAST timeout="300,600,1200,2400,3600"/>
          <pbcast.STABLE stability_delay="1000" desired_avg_gossip="50000"
                   max_bytes="400000"/>
          <pbcast.GMS print_local_addr="true" join_timeout="3000"
                   shun="true"
                   view_bundling="true"
                   view_ack_collection_timeout="5000"/>
          <FC max_credits="500000" min_threshold="0.20" ignore_synchronous_response="true"/>
          <FRAG2 frag_size="60000"/>
          <!-- pbcast.STREAMING_STATE_TRANSFER/ -->
          <pbcast.STATE_TRANSFER/>
          <pbcast.FLUSH timeout="0"/>
        </config>

    </stack>

What is wrong with my configuration?  Any hint will be greatly appreciated.

TIA