Removing nodes breaks further cluster formation with "ISPN000172 Failed to prepare view" ---------------------------------------------------------------------------------------- Key: AS7-3166 URL: https://issues.jboss.org/browse/AS7-3166 Project: Application Server 7 Issue Type: Feature Request Components: Clustering Affects Versions: 7.1.0.CR1 Reporter: Radoslav Husar Assignee: Paul Ferraro Priority: Critical Labels: failover_testing Fix For: 7.1.0.Final when 2 nodes from 3 node cluster are shutdown (nearly at the same time), the remaining nodes keeps logging every minute {noformat} 13:30:25,475 ERROR [org.infinispan.cacheviews.CacheViewsManagerImpl] (CacheViewInstaller-1,s2/web) ISPN000172: Failed to prepare view CacheView{viewId=42, members=[s2/web, rhusar/web]} for cache registry, rolling back to view CacheView{viewId=41, members=[s2/web]}: java.util.concurrent.ExecutionException: org.infinispan.remoting.transport.jgroups.SuspectException: One or more nodes have left the cluster while replicating command CacheViewControlCommand{cache=registry, type=PREPARE_VIEW, sender=s2/web, newViewId=42, newMembers=[s2/web, rhusar/web], oldViewId=41, oldMembers=[s2/web]} at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:232) [:1.6.0_29] at java.util.concurrent.FutureTask.get(FutureTask.java:91) [:1.6.0_29] at org.infinispan.cacheviews.CacheViewsManagerImpl.clusterPrepareView(CacheViewsManagerImpl.java:305) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.cacheviews.CacheViewsManagerImpl.clusterInstallView(CacheViewsManagerImpl.java:251) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.cacheviews.CacheViewsManagerImpl$ViewInstallationTask.call(CacheViewsManagerImpl.java:831) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [:1.6.0_29] at java.util.concurrent.FutureTask.run(FutureTask.java:138) [:1.6.0_29] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [:1.6.0_29] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [:1.6.0_29] at java.lang.Thread.run(Thread.java:662) [:1.6.0_29] Caused by: org.infinispan.remoting.transport.jgroups.SuspectException: One or more nodes have left the cluster while replicating command CacheViewControlCommand{cache=registry, type=PREPARE_VIEW, sender=s2/web, newViewId=42, newMembers=[s2/web, rhusar/web], oldViewId=41, oldMembers=[s2/web]} at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:404) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.cacheviews.CacheViewsManagerImpl$2.call(CacheViewsManagerImpl.java:294) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.cacheviews.CacheViewsManagerImpl$2.call(CacheViewsManagerImpl.java:291) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] ... 5 more 13:30:25,476 ERROR [org.infinispan.cacheviews.CacheViewsManagerImpl] (CacheViewInstaller-2,s2/web) ISPN000172: Failed to prepare view CacheView{viewId=40, members=[s2/web, rhusar/web]} for cache repl, rolling back to view CacheView{viewId=39, members=[s2/web]}: java.util.concurrent.ExecutionException: org.infinispan.remoting.transport.jgroups.SuspectException: One or more nodes have left the cluster while replicating command CacheViewControlCommand{cache=repl, type=PREPARE_VIEW, sender=s2/web, newViewId=40, newMembers=[s2/web, rhusar/web], oldViewId=39, oldMembers=[s2/web]} at java.util.concurrent.FutureTask$Sync.innerGet(FutureTask.java:232) [:1.6.0_29] at java.util.concurrent.FutureTask.get(FutureTask.java:91) [:1.6.0_29] at org.infinispan.cacheviews.CacheViewsManagerImpl.clusterPrepareView(CacheViewsManagerImpl.java:305) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.cacheviews.CacheViewsManagerImpl.clusterInstallView(CacheViewsManagerImpl.java:251) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.cacheviews.CacheViewsManagerImpl$ViewInstallationTask.call(CacheViewsManagerImpl.java:831) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [:1.6.0_29] at java.util.concurrent.FutureTask.run(FutureTask.java:138) [:1.6.0_29] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [:1.6.0_29] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [:1.6.0_29] at java.lang.Thread.run(Thread.java:662) [:1.6.0_29] Caused by: org.infinispan.remoting.transport.jgroups.SuspectException: One or more nodes have left the cluster while replicating command CacheViewControlCommand{cache=repl, type=PREPARE_VIEW, sender=s2/web, newViewId=40, newMembers=[s2/web, rhusar/web], oldViewId=39, oldMembers=[s2/web]} at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:404) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.cacheviews.CacheViewsManagerImpl$2.call(CacheViewsManagerImpl.java:294) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.cacheviews.CacheViewsManagerImpl$2.call(CacheViewsManagerImpl.java:291) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] ... 5 more 13:30:25,484 WARNING [org.jgroups.protocols.UDP] (CacheViewInstaller-2,s2/web) null: no physical address for rhusar/web, dropping message 13:30:25,484 WARNING [org.jgroups.protocols.UDP] (CacheViewInstaller-1,s2/web) null: no physical address for rhusar/web, dropping message {noformat} while 1st node terminated correctly, and 2nd one with a warning {noformat} 13:13:06,128 WARN [org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher] Problems unmarshalling remote command from byte buffer: org.infinispan.CacheException: Cache manager is either starting up or shutting down but it's not interrupted, so type (id=74) cannot be resolved. at org.infinispan.marshall.jboss.ExternalizerTable.readObject(ExternalizerTable.java:257) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:37) [jboss-marshalling-1.3.4.GA.jar:1.3.4.GA] at org.infinispan.marshall.jboss.AbstractJBossMarshaller.objectFromObjectStream(AbstractJBossMarshaller.java:120) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.marshall.VersionAwareMarshaller.objectFromByteBuffer(VersionAwareMarshaller.java:115) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.marshall.AbstractDelegatingMarshaller.objectFromByteBuffer(AbstractDelegatingMarshaller.java:79) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectFromBuffer(MarshallerAdapter.java:50) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:139) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:447) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:354) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:230) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:556) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jboss.as.clustering.jgroups.ClassLoaderAwareUpHandler.up(ClassLoaderAwareUpHandler.java:56) [jboss-as-clustering-jgroups-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jgroups.blocks.mux.MuxUpHandler.up(MuxUpHandler.java:130) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jboss.as.clustering.jgroups.MuxChannel$ClassLoaderAwareMuxUpHandler.up(MuxChannel.java:64) [jboss-as-clustering-jgroups-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jgroups.JChannel.up(JChannel.java:716) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1026) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:433) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:178) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.FRAG2.up(FRAG2.java:181) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:418) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:418) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:881) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:244) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.UNICAST2.handleDataReceived(UNICAST2.java:759) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.UNICAST2.up(UNICAST2.java:365) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:597) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.BARRIER.up(BARRIER.java:102) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:140) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.FD.up(FD.java:273) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:284) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.MERGE2.up(MERGE2.java:205) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.Discovery.up(Discovery.java:354) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.stack.Protocol.up(Protocol.java:358) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.TP.passMessageUp(TP.java:1174) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1709) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1691) [jgroups-3.0.1.Final.jar:3.0.1.Final] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [:1.6.0_29] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [:1.6.0_29] at java.lang.Thread.run(Thread.java:662) [:1.6.0_29] 13:13:06,131 WARN [org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher] Problems unmarshalling remote command from byte buffer: org.infinispan.CacheException: Cache manager is either starting up or shutting down but it's not interrupted, so type (id=74) cannot be resolved. at org.infinispan.marshall.jboss.ExternalizerTable.readObject(ExternalizerTable.java:257) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351) at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209) at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:37) [jboss-marshalling-1.3.4.GA.jar:1.3.4.GA] at org.infinispan.marshall.jboss.AbstractJBossMarshaller.objectFromObjectStream(AbstractJBossMarshaller.java:120) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.marshall.VersionAwareMarshaller.objectFromByteBuffer(VersionAwareMarshaller.java:115) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.marshall.AbstractDelegatingMarshaller.objectFromByteBuffer(AbstractDelegatingMarshaller.java:79) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectFromBuffer(MarshallerAdapter.java:50) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.handle(CommandAwareRpcDispatcher.java:139) [infinispan-core-5.1.0.CR1.jar:5.1.0.CR1] at org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:447) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:354) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:230) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:556) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jboss.as.clustering.jgroups.ClassLoaderAwareUpHandler.up(ClassLoaderAwareUpHandler.java:56) [jboss-as-clustering-jgroups-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jgroups.blocks.mux.MuxUpHandler.up(MuxUpHandler.java:130) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jboss.as.clustering.jgroups.MuxChannel$ClassLoaderAwareMuxUpHandler.up(MuxChannel.java:64) [jboss-as-clustering-jgroups-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jgroups.JChannel.up(JChannel.java:716) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1026) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.pbcast.FLUSH.up(FLUSH.java:433) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:178) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.FRAG2.up(FRAG2.java:181) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:418) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.FlowControl.up(FlowControl.java:418) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.pbcast.GMS.up(GMS.java:881) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:244) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.UNICAST2.handleDataReceived(UNICAST2.java:759) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.UNICAST2.up(UNICAST2.java:365) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:597) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.BARRIER.up(BARRIER.java:102) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:140) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.FD.up(FD.java:273) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:284) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.MERGE2.up(MERGE2.java:205) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.Discovery.up(Discovery.java:354) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.stack.Protocol.up(Protocol.java:358) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.TP.passMessageUp(TP.java:1174) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1709) [jgroups-3.0.1.Final.jar:3.0.1.Final] at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1691) [jgroups-3.0.1.Final.jar:3.0.1.Final] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [:1.6.0_29] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [:1.6.0_29] at java.lang.Thread.run(Thread.java:662) [:1.6.0_29] 13:13:06,150 INFO [org.infinispan.remoting.transport.jgroups.JGroupsTransport] ISPN000082: Stopping the RpcDispatcher 13:13:06,158 INFO [com.arjuna.ats.jbossatx] ARJUNA032018: Destroying TransactionManagerService 13:13:06,159 INFO [com.arjuna.ats.jbossatx] ARJUNA032014: Stopping transaction recovery manager 13:13:06,488 INFO [org.jboss.as] JBoss AS 7.1.0.Final-SNAPSHOT "Flux Capacitor" stopped in 961ms {noformat} nodes are then unable to join back in, deployments are rolled back after minutes {noformat} 13:35:02,755 INFO [org.jboss.as.server] (DeploymentScanner-threads - 2) JBAS015856: Undeploy of deployment "clusterbench-ee6.ear" was rolled back with failure message Operation cancelled 13:35:02,757 ERROR [org.jboss.as.controller.management-operation] (DeploymentScanner-threads - 2) JBAS014612: Operation ("deploy") failed - address: ([("deployment" => "clusterbench-ee6.ear")]): java.util.concurrent.CancellationException: Operation cancelled asynchronously at org.jboss.as.controller.OperationContextImpl.awaitContainerMonitor(OperationContextImpl.java:356) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.OperationContextImpl.releaseStepLocks(OperationContextImpl.java:599) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext$Step.finalizeInternal(AbstractOperationContext.java:620) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext$Step.finalizeStep(AbstractOperationContext.java:579) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext$Step.access$200(AbstractOperationContext.java:560) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.executeStep(AbstractOperationContext.java:429) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.doCompleteStep(AbstractOperationContext.java:254) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.completeStep(AbstractOperationContext.java:190) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.server.deployment.DeploymentDeployHandler.execute(DeploymentDeployHandler.java:68) [jboss-as-server-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.executeStep(AbstractOperationContext.java:359) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.doCompleteStep(AbstractOperationContext.java:254) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.completeStep(AbstractOperationContext.java:190) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.server.deployment.DeploymentAddHandler.execute(DeploymentAddHandler.java:185) [jboss-as-server-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.executeStep(AbstractOperationContext.java:359) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.doCompleteStep(AbstractOperationContext.java:254) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.completeStep(AbstractOperationContext.java:190) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.CompositeOperationHandler.execute(CompositeOperationHandler.java:84) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.executeStep(AbstractOperationContext.java:359) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.doCompleteStep(AbstractOperationContext.java:254) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.completeStep(AbstractOperationContext.java:190) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.CompositeOperationHandler.execute(CompositeOperationHandler.java:84) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.executeStep(AbstractOperationContext.java:359) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.doCompleteStep(AbstractOperationContext.java:254) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.completeStep(AbstractOperationContext.java:190) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.ModelControllerImpl$DefaultPrepareStepHandler.execute(ModelControllerImpl.java:432) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.executeStep(AbstractOperationContext.java:359) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.doCompleteStep(AbstractOperationContext.java:254) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.AbstractOperationContext.completeStep(AbstractOperationContext.java:190) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.ModelControllerImpl.execute(ModelControllerImpl.java:119) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.ModelControllerImpl$1.execute(ModelControllerImpl.java:302) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.controller.ModelControllerImpl$1.execute(ModelControllerImpl.java:292) [jboss-as-controller-7.1.0.Final-SNAPSHOT.jar:7.1.0.Final-SNAPSHOT] at org.jboss.as.server.deployment.scanner.FileSystemDeploymentService$DeploymentTask.call(FileSystemDeploymentService.java:1178) at org.jboss.as.server.deployment.scanner.FileSystemDeploymentService$DeploymentTask.call(FileSystemDeploymentService.java:1168) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [:1.6.0_29] at java.util.concurrent.FutureTask.run(FutureTask.java:138) [:1.6.0_29] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:98) [:1.6.0_29] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:206) [:1.6.0_29] at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [:1.6.0_29] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [:1.6.0_29] at java.lang.Thread.run(Thread.java:662) [:1.6.0_29] at org.jboss.threads.JBossThread.run(JBossThread.java:122) [jboss-threads-2.0.0.GA.jar:2.0.0.GA] {noformat}