Singleton service jboss.test.myservice is not started ----------------------------------------------------- Key: WFLY-7982 URL: https://issues.jboss.org/browse/WFLY-7982 Project: WildFly Issue Type: Bug Components: Clustering Affects Versions: 10.1.0.Final Reporter: Paul Ferraro Assignee: Paul Ferraro We saw 5 occurences of an error on *server* during failover scenario *eap-7x-failover-jvmkill-ha-ss* (jvmkill scenario with HA singleton service). Timeline: - 18:46:02,901 perf20 started - 18:46:09,303 perf20 received new cluster view - 18:51:04,377 perf20 elected as the singleton provider of the jboss.test.myservice service - 18:51:04,386 singleton service started on perf20 Server log stacktrace: {code} 18:51:04,386 INFO [org.jboss.test.singletonservice.TestingSingletonService] (MSC service thread 1-1) Service started on perf20 [JBossINF] 18:51:04,388 ERROR [org.jboss.as.ejb3.invocation] (default task-75) WFLYEJB0034: EJB Invocation failed on component ServiceAccessBean for method public abstract java.lang.String org.jboss.test.singletonservice.ServiceAccess.getNodeNameOfService(): javax.ejb.EJBException: java.lang.IllegalStateException: WFLYCLSV0009: Singleton service jboss.test.myservice is not started. [JBossINF] at org.jboss.as.ejb3.tx.CMTTxInterceptor.handleExceptionInOurTx(CMTTxInterceptor.java:187) [JBossINF] at org.jboss.as.ejb3.tx.CMTTxInterceptor.invokeInOurTx(CMTTxInterceptor.java:277) [JBossINF] at org.jboss.as.ejb3.tx.CMTTxInterceptor.required(CMTTxInterceptor.java:327) [JBossINF] at org.jboss.as.ejb3.tx.CMTTxInterceptor.processInvocation(CMTTxInterceptor.java:239) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.invocation.InterceptorContext$Invocation.proceed(InterceptorContext.java:456) [JBossINF] at org.jboss.weld.ejb.AbstractEJBRequestScopeActivationInterceptor.aroundInvoke(AbstractEJBRequestScopeActivationInterceptor.java:73) [JBossINF] at org.jboss.as.weld.ejb.EjbRequestScopeActivationInterceptor.processInvocation(EjbRequestScopeActivationInterceptor.java:84) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.remote.EJBRemoteTransactionPropagatingInterceptor.processInvocation(EJBRemoteTransactionPropagatingInterceptor.java:80) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.component.interceptors.CurrentInvocationContextInterceptor.processInvocation(CurrentInvocationContextInterceptor.java:41) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.component.invocationmetrics.WaitTimeInterceptor.processInvocation(WaitTimeInterceptor.java:47) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.security.SecurityContextInterceptor.processInvocation(SecurityContextInterceptor.java:100) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.deployment.processors.StartupAwaitInterceptor.processInvocation(StartupAwaitInterceptor.java:22) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.component.interceptors.ShutDownInterceptorFactory$1.processInvocation(ShutDownInterceptorFactory.java:64) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.deployment.processors.EjbSuspendInterceptor.processInvocation(EjbSuspendInterceptor.java:53) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.component.interceptors.LoggingInterceptor.processInvocation(LoggingInterceptor.java:67) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ee.component.NamespaceContextInterceptor.processInvocation(NamespaceContextInterceptor.java:50) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.component.interceptors.AdditionalSetupInterceptor.processInvocation(AdditionalSetupInterceptor.java:54) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.invocation.ContextClassLoaderInterceptor.processInvocation(ContextClassLoaderInterceptor.java:64) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:375) [JBossINF] at org.wildfly.security.manager.WildFlySecurityManager.doChecked(WildFlySecurityManager.java:609) [JBossINF] at org.jboss.invocation.AccessCheckingInterceptor.processInvocation(AccessCheckingInterceptor.java:61) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:375) [JBossINF] at org.jboss.invocation.PrivilegedWithCombinerInterceptor.processInvocation(PrivilegedWithCombinerInterceptor.java:80) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.invocation.ChainedInterceptor.processInvocation(ChainedInterceptor.java:61) [JBossINF] at org.jboss.as.ee.component.ViewService$View.invoke(ViewService.java:198) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.invokeMethod(MethodInvocationMessageHandler.java:340) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.access$100(MethodInvocationMessageHandler.java:68) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler$1.run(MethodInvocationMessageHandler.java:202) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.processMessage(MethodInvocationMessageHandler.java:268) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver.processMessage(VersionOneProtocolChannelReceiver.java:213) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versiontwo.VersionTwoProtocolChannelReceiver.processMessage(VersionTwoProtocolChannelReceiver.java:76) [JBossINF] at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver.handleMessage(VersionOneProtocolChannelReceiver.java:159) [JBossINF] at org.jboss.remoting3.remote.RemoteConnectionChannel.lambda$handleMessageData$3(RemoteConnectionChannel.java:434) [JBossINF] at org.jboss.remoting3.EndpointImpl$TrackingExecutor.lambda$execute$0(EndpointImpl.java:789) [JBossINF] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [JBossINF] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [JBossINF] at java.lang.Thread.run(Thread.java:745) [JBossINF] Caused by: java.lang.IllegalStateException: WFLYCLSV0009: Singleton service jboss.test.myservice is not started. [JBossINF] at org.wildfly.clustering.server.singleton.PrimaryProxyService.getValue(PrimaryProxyService.java:71) [JBossINF] at org.jboss.msc.service.ServiceControllerImpl.getValue(ServiceControllerImpl.java:1159) [JBossINF] at org.wildfly.clustering.server.singleton.DistributedSingletonService.getValue(DistributedSingletonService.java:177) [JBossINF] at org.wildfly.clustering.service.AsynchronousServiceBuilder.getValue(AsynchronousServiceBuilder.java:91) [JBossINF] at org.jboss.msc.service.ServiceControllerImpl.getValue(ServiceControllerImpl.java:1159) [JBossINF] at org.jboss.test.singletonservice.ServiceAccessBean.getNodeNameOfService(ServiceAccessBean.java:20) [JBossINF] at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) [JBossINF] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [JBossINF] at java.lang.reflect.Method.invoke(Method.java:498) [JBossINF] at org.jboss.as.ee.component.ManagedReferenceMethodInterceptor.processInvocation(ManagedReferenceMethodInterceptor.java:52) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.invocation.InterceptorContext$Invocation.proceed(InterceptorContext.java:456) [JBossINF] at org.jboss.as.weld.interceptors.Jsr299BindingsInterceptor.doMethodInterception(Jsr299BindingsInterceptor.java:85) [JBossINF] at org.jboss.as.weld.interceptors.Jsr299BindingsInterceptor.processInvocation(Jsr299BindingsInterceptor.java:96) [JBossINF] at org.jboss.as.ee.component.interceptors.UserInterceptorFactory$1.processInvocation(UserInterceptorFactory.java:63) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.component.invocationmetrics.ExecutionTimeInterceptor.processInvocation(ExecutionTimeInterceptor.java:43) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.jpa.interceptor.SBInvocationInterceptor.processInvocation(SBInvocationInterceptor.java:47) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ee.concurrent.ConcurrentContextInterceptor.processInvocation(ConcurrentContextInterceptor.java:45) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.invocation.InitialInterceptor.processInvocation(InitialInterceptor.java:21) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.invocation.ChainedInterceptor.processInvocation(ChainedInterceptor.java:61) [JBossINF] at org.jboss.as.ee.component.interceptors.ComponentDispatcherInterceptor.processInvocation(ComponentDispatcherInterceptor.java:52) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.component.pool.PooledInstanceInterceptor.processInvocation(PooledInstanceInterceptor.java:51) [JBossINF] at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) [JBossINF] at org.jboss.as.ejb3.tx.CMTTxInterceptor.invokeInOurTx(CMTTxInterceptor.java:275) [JBossINF] ... 50 more {code} Singleton service jboss.test.myservice not started error occured also on client at the same time: {code} 2017/01/12 18:51:04:396 EST [DEBUG][Runner - 1] HOST perf17.mw.lab.eng.bos.redhat.com:rootProcess:c - Response acquired, node name: perf20 2017/01/12 18:51:04:582 EST [ERROR][Runner - 3] HOST perf17.mw.lab.eng.bos.redhat.com:rootProcess:c - Error getting node name! <javax.ejb.EJBException: java.lang.IllegalStateException: WFLYCLSV0009: Singleton service jboss.test.myservice is not started.> javax.ejb.EJBException: java.lang.IllegalStateException: WFLYCLSV0009: Singleton service jboss.test.myservice is not started. at org.jboss.as.ejb3.tx.CMTTxInterceptor.handleExceptionInOurTx(CMTTxInterceptor.java:187) at org.jboss.as.ejb3.tx.CMTTxInterceptor.invokeInOurTx(CMTTxInterceptor.java:277) at org.jboss.as.ejb3.tx.CMTTxInterceptor.required(CMTTxInterceptor.java:327) at org.jboss.as.ejb3.tx.CMTTxInterceptor.processInvocation(CMTTxInterceptor.java:239) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.InterceptorContext$Invocation.proceed(InterceptorContext.java:456) at org.jboss.weld.ejb.AbstractEJBRequestScopeActivationInterceptor.aroundInvoke(AbstractEJBRequestScopeActivationInterceptor.java:73) at org.jboss.as.weld.ejb.EjbRequestScopeActivationInterceptor.processInvocation(EjbRequestScopeActivationInterceptor.java:84) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.remote.EJBRemoteTransactionPropagatingInterceptor.processInvocation(EJBRemoteTransactionPropagatingInterceptor.java:80) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.interceptors.CurrentInvocationContextInterceptor.processInvocation(CurrentInvocationContextInterceptor.java:41) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.invocationmetrics.WaitTimeInterceptor.processInvocation(WaitTimeInterceptor.java:47) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.security.SecurityContextInterceptor.processInvocation(SecurityContextInterceptor.java:100) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.deployment.processors.StartupAwaitInterceptor.processInvocation(StartupAwaitInterceptor.java:22) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.interceptors.ShutDownInterceptorFactory$1.processInvocation(ShutDownInterceptorFactory.java:64) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.deployment.processors.EjbSuspendInterceptor.processInvocation(EjbSuspendInterceptor.java:53) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.interceptors.LoggingInterceptor.processInvocation(LoggingInterceptor.java:67) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ee.component.NamespaceContextInterceptor.processInvocation(NamespaceContextInterceptor.java:50) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.interceptors.AdditionalSetupInterceptor.processInvocation(AdditionalSetupInterceptor.java:54) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.ContextClassLoaderInterceptor.processInvocation(ContextClassLoaderInterceptor.java:64) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:375) at org.wildfly.security.manager.WildFlySecurityManager.doChecked(WildFlySecurityManager.java:609) at org.jboss.invocation.AccessCheckingInterceptor.processInvocation(AccessCheckingInterceptor.java:61) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:375) at org.jboss.invocation.PrivilegedWithCombinerInterceptor.processInvocation(PrivilegedWithCombinerInterceptor.java:80) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.ChainedInterceptor.processInvocation(ChainedInterceptor.java:61) at org.jboss.as.ee.component.ViewService$View.invoke(ViewService.java:198) at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.invokeMethod(MethodInvocationMessageHandler.java:340) at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.access$100(MethodInvocationMessageHandler.java:68) at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler$1.run(MethodInvocationMessageHandler.java:202) at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.processMessage(MethodInvocationMessageHandler.java:268) at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver.processMessage(VersionOneProtocolChannelReceiver.java:213) at org.jboss.as.ejb3.remote.protocol.versiontwo.VersionTwoProtocolChannelReceiver.processMessage(VersionTwoProtocolChannelReceiver.java:76) at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver.handleMessage(VersionOneProtocolChannelReceiver.java:159) at org.jboss.remoting3.remote.RemoteConnectionChannel.lambda$handleMessageData$3(RemoteConnectionChannel.java:434) at org.jboss.remoting3.EndpointImpl$TrackingExecutor.lambda$execute$0(EndpointImpl.java:789) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) at ...asynchronous invocation...(Unknown Source) at org.jboss.ejb.client.remoting.InvocationExceptionResponseHandler$MethodInvocationExceptionResultProducer.getResult(InvocationExceptionResponseHandler.java:96) at org.jboss.ejb.client.EJBClientInvocationContext$1.run(EJBClientInvocationContext.java:282) at java.security.AccessController.doPrivileged(Native Method) at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:279) at org.jboss.ejb.client.EJBObjectInterceptor.handleInvocationResult(EJBObjectInterceptor.java:64) at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:290) at org.jboss.ejb.client.EJBHomeInterceptor.handleInvocationResult(EJBHomeInterceptor.java:88) at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:290) at org.jboss.ejb.client.TransactionInterceptor.handleInvocationResult(TransactionInterceptor.java:46) at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:290) at org.jboss.ejb.client.ReceiverInterceptor.handleInvocationResult(ReceiverInterceptor.java:142) at org.jboss.ejb.client.EJBClientInvocationContext.getResult(EJBClientInvocationContext.java:265) at org.jboss.ejb.client.EJBClientInvocationContext.awaitResponse(EJBClientInvocationContext.java:453) at org.jboss.ejb.client.EJBInvocationHandler.doInvoke(EJBInvocationHandler.java:204) at org.jboss.ejb.client.EJBInvocationHandler.doInvoke(EJBInvocationHandler.java:183) at org.jboss.ejb.client.EJBInvocationHandler.invoke(EJBInvocationHandler.java:146) at com.sun.proxy.$Proxy0.getNodeNameOfService(Unknown Source) at org.jboss.smartfrog.clustering.ejb3.SSAccessProcessorFactoryImpl$SSAccessRequestProcessor.processRequest(SSAccessProcessorFactoryImpl.java:88) at org.jboss.smartfrog.loaddriver.CompoundRequestProcessorFactoryImpl$CompoundRequestProcessor.processRequest(CompoundRequestProcessorFactoryImpl.java:52) at org.jboss.smartfrog.loaddriver.Runner.run(Runner.java:103) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.IllegalStateException: WFLYCLSV0009: Singleton service jboss.test.myservice is not started. at org.wildfly.clustering.server.singleton.PrimaryProxyService.getValue(PrimaryProxyService.java:71) at org.jboss.msc.service.ServiceControllerImpl.getValue(ServiceControllerImpl.java:1159) at org.wildfly.clustering.server.singleton.DistributedSingletonService.getValue(DistributedSingletonService.java:177) at org.wildfly.clustering.service.AsynchronousServiceBuilder.getValue(AsynchronousServiceBuilder.java:91) at org.jboss.msc.service.ServiceControllerImpl.getValue(ServiceControllerImpl.java:1159) at org.jboss.test.singletonservice.ServiceAccessBean.getNodeNameOfService(ServiceAccessBean.java:20) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.jboss.as.ee.component.ManagedReferenceMethodInterceptor.processInvocation(ManagedReferenceMethodInterceptor.java:52) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.InterceptorContext$Invocation.proceed(InterceptorContext.java:456) at org.jboss.as.weld.interceptors.Jsr299BindingsInterceptor.doMethodInterception(Jsr299BindingsInterceptor.java:85) at org.jboss.as.weld.interceptors.Jsr299BindingsInterceptor.processInvocation(Jsr299BindingsInterceptor.java:96) at org.jboss.as.ee.component.interceptors.UserInterceptorFactory$1.processInvocation(UserInterceptorFactory.java:63) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.invocationmetrics.ExecutionTimeInterceptor.processInvocation(ExecutionTimeInterceptor.java:43) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.jpa.interceptor.SBInvocationInterceptor.processInvocation(SBInvocationInterceptor.java:47) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ee.concurrent.ConcurrentContextInterceptor.processInvocation(ConcurrentContextInterceptor.java:45) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.InitialInterceptor.processInvocation(InitialInterceptor.java:21) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.ChainedInterceptor.processInvocation(ChainedInterceptor.java:61) at org.jboss.as.ee.component.interceptors.ComponentDispatcherInterceptor.processInvocation(ComponentDispatcherInterceptor.java:52) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.pool.PooledInstanceInterceptor.processInvocation(PooledInstanceInterceptor.java:51) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.tx.CMTTxInterceptor.invokeInOurTx(CMTTxInterceptor.java:275) at org.jboss.as.ejb3.tx.CMTTxInterceptor.required(CMTTxInterceptor.java:327) at org.jboss.as.ejb3.tx.CMTTxInterceptor.processInvocation(CMTTxInterceptor.java:239) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.InterceptorContext$Invocation.proceed(InterceptorContext.java:456) at org.jboss.weld.ejb.AbstractEJBRequestScopeActivationInterceptor.aroundInvoke(AbstractEJBRequestScopeActivationInterceptor.java:73) at org.jboss.as.weld.ejb.EjbRequestScopeActivationInterceptor.processInvocation(EjbRequestScopeActivationInterceptor.java:84) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.remote.EJBRemoteTransactionPropagatingInterceptor.processInvocation(EJBRemoteTransactionPropagatingInterceptor.java:80) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.interceptors.CurrentInvocationContextInterceptor.processInvocation(CurrentInvocationContextInterceptor.java:41) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.invocationmetrics.WaitTimeInterceptor.processInvocation(WaitTimeInterceptor.java:47) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.security.SecurityContextInterceptor.processInvocation(SecurityContextInterceptor.java:100) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.deployment.processors.StartupAwaitInterceptor.processInvocation(StartupAwaitInterceptor.java:22) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.interceptors.ShutDownInterceptorFactory$1.processInvocation(ShutDownInterceptorFactory.java:64) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.deployment.processors.EjbSuspendInterceptor.processInvocation(EjbSuspendInterceptor.java:53) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.interceptors.LoggingInterceptor.processInvocation(LoggingInterceptor.java:67) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ee.component.NamespaceContextInterceptor.processInvocation(NamespaceContextInterceptor.java:50) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.as.ejb3.component.interceptors.AdditionalSetupInterceptor.processInvocation(AdditionalSetupInterceptor.java:54) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.ContextClassLoaderInterceptor.processInvocation(ContextClassLoaderInterceptor.java:64) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:375) at org.wildfly.security.manager.WildFlySecurityManager.doChecked(WildFlySecurityManager.java:609) at org.jboss.invocation.AccessCheckingInterceptor.processInvocation(AccessCheckingInterceptor.java:61) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:375) at org.jboss.invocation.PrivilegedWithCombinerInterceptor.processInvocation(PrivilegedWithCombinerInterceptor.java:80) at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:359) at org.jboss.invocation.ChainedInterceptor.processInvocation(ChainedInterceptor.java:61) at org.jboss.as.ee.component.ViewService$View.invoke(ViewService.java:198) at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.invokeMethod(MethodInvocationMessageHandler.java:340) at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.access$100(MethodInvocationMessageHandler.java:68) at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler$1.run(MethodInvocationMessageHandler.java:202) at org.jboss.as.ejb3.remote.protocol.versionone.MethodInvocationMessageHandler.processMessage(MethodInvocationMessageHandler.java:268) at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver.processMessage(VersionOneProtocolChannelReceiver.java:213) at org.jboss.as.ejb3.remote.protocol.versiontwo.VersionTwoProtocolChannelReceiver.processMessage(VersionTwoProtocolChannelReceiver.java:76) at org.jboss.as.ejb3.remote.protocol.versionone.VersionOneProtocolChannelReceiver.handleMessage(VersionOneProtocolChannelReceiver.java:159) at org.jboss.remoting3.remote.RemoteConnectionChannel.lambda$handleMessageData$3(RemoteConnectionChannel.java:434) at org.jboss.remoting3.EndpointImpl$TrackingExecutor.lambda$execute$0(EndpointImpl.java:789) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ... 1 more 2017/01/12 18:51:04:584 EST [INFO ][Runner - 3] HOST perf17.mw.lab.eng.bos.redhat.com:rootProcess:c - Resettin {code} Link to server log: http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/perflab_eap-7x-failov... Link to client log: http://jenkins.mw.lab.eng.bos.redhat.com/hudson/job/perflab_eap-7x-failov...