[jboss-cvs] JBoss Messaging SVN: r5123 - in trunk: src/main/org/jboss/messaging/core/client/impl and 16 other directories.

jboss-cvs-commits at lists.jboss.org jboss-cvs-commits at lists.jboss.org
Thu Oct 16 15:14:12 EDT 2008


Author: timfox
Date: 2008-10-16 15:14:11 -0400 (Thu, 16 Oct 2008)
New Revision: 5123

Added:
   trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionAcknowledgeMessage.java
   trunk/tests/src/org/jboss/messaging/tests/integration/cluster/MultiThreadRandomFailoverTest.java
   trunk/tests/src/org/jboss/messaging/tests/soak/
   trunk/tests/src/org/jboss/messaging/tests/soak/failover/
   trunk/tests/src/org/jboss/messaging/tests/soak/failover/RandomFailoverSoakTest.java
Removed:
   trunk/src/main/org/jboss/messaging/core/remoting/ResponseNotifier.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionProcessedMessage.java
   trunk/tests/src/org/jboss/messaging/tests/integration/cluster/SystematicFailoverTest.java
Modified:
   trunk/build-messaging.xml
   trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerImpl.java
   trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerInternal.java
   trunk/src/main/org/jboss/messaging/core/client/impl/ClientMessageImpl.java
   trunk/src/main/org/jboss/messaging/core/client/impl/ClientProducerImpl.java
   trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionFactoryImpl.java
   trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionFactoryInternal.java
   trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionImpl.java
   trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionInternal.java
   trunk/src/main/org/jboss/messaging/core/postoffice/PostOffice.java
   trunk/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java
   trunk/src/main/org/jboss/messaging/core/postoffice/impl/SimpleAddressManager.java
   trunk/src/main/org/jboss/messaging/core/postoffice/impl/WildcardAddressManager.java
   trunk/src/main/org/jboss/messaging/core/remoting/Channel.java
   trunk/src/main/org/jboss/messaging/core/remoting/Packet.java
   trunk/src/main/org/jboss/messaging/core/remoting/RemotingConnection.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/ConnectionRegistryImpl.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/RemotingConnectionImpl.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/RemotingServiceImpl.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/invm/InVMConnection.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/PacketImpl.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionCreateQueueMessage.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionDeleteQueueMessage.java
   trunk/src/main/org/jboss/messaging/core/server/ServerConsumer.java
   trunk/src/main/org/jboss/messaging/core/server/ServerSession.java
   trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java
   trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerPacketHandler.java
   trunk/src/main/org/jboss/messaging/core/server/impl/QueueImpl.java
   trunk/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java
   trunk/src/main/org/jboss/messaging/core/server/impl/ServerProducerImpl.java
   trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java
   trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionPacketHandler.java
   trunk/src/main/org/jboss/messaging/core/transaction/ResourceManager.java
   trunk/src/main/org/jboss/messaging/core/transaction/Transaction.java
   trunk/src/main/org/jboss/messaging/core/transaction/impl/ResourceManagerImpl.java
   trunk/src/main/org/jboss/messaging/core/transaction/impl/TransactionImpl.java
   trunk/src/main/org/jboss/messaging/util/OrderedExecutorFactory.java
   trunk/src/main/org/jboss/messaging/util/VersionLoader.java
   trunk/tests/jms-tests/src/org/jboss/test/messaging/jms/AcknowledgementTest.java
   trunk/tests/src/org/jboss/messaging/tests/integration/cluster/RandomFailoverTest.java
   trunk/tests/src/org/jboss/messaging/tests/integration/cluster/SimpleManualFailoverTest.java
Log:
More session replication and failover


Modified: trunk/build-messaging.xml
===================================================================
--- trunk/build-messaging.xml	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/build-messaging.xml	2008-10-16 19:14:11 UTC (rev 5123)
@@ -701,8 +701,6 @@
             <formatter type="plain" usefile="${junit.formatter.usefile}"/>
             <fileset dir="${test.classes.dir}">
                <include name="${tests.param}"/>
-               <!-- Temporarily exclude cluster tests until I have fixed them -->
-               <exclude name="**/tests/integration/cluster/**/*.class"/>
             </fileset>
          </batchtest>
       </junit>

Modified: trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -58,7 +58,7 @@
    private final Executor sessionExecutor;
 
    private final int clientWindowSize;
- 
+
    private final Queue<ClientMessage> buffer = new LinkedList<ClientMessage>();
 
    private final boolean direct;
@@ -76,11 +76,9 @@
    private volatile boolean closed;
 
    private volatile int creditsToSend;
-   
-   private boolean cleared;
 
    private boolean messagesWaiting = true;
-   
+
    // Constructors
    // ---------------------------------------------------------------------------------
 
@@ -89,7 +87,8 @@
                              final int clientWindowSize,
                              final boolean direct,
                              final Executor executor,
-                             final Channel channel, final boolean isBrowser)
+                             final Channel channel,
+                             final boolean isBrowser)
    {
       this.id = id;
 
@@ -163,8 +162,8 @@
 
                if (expired)
                {
-                  session.processed(id, m.getMessageID());
-                  
+                  session.acknowledge(id, m.getMessageID());
+
                   if (toWait > 0)
                   {
                      continue;
@@ -173,7 +172,7 @@
                   {
                      return null;
                   }
-               }               
+               }
 
                return m;
             }
@@ -210,7 +209,7 @@
    {
       checkClosed();
 
-      if(isBrowser)
+      if (isBrowser)
       {
          throw new MessagingException(MessagingException.ILLEGAL_STATE,
                                       "Cannot set MessageHandler - consumer is in browser mode");
@@ -240,7 +239,7 @@
       doCleanUp(true);
    }
 
-   public synchronized void cleanUp()
+   public void cleanUp()
    {
       try
       {
@@ -279,20 +278,20 @@
     */
    public boolean awaitMessage(long timeOut) throws Exception
    {
-      if(!buffer.isEmpty())
+      if (!buffer.isEmpty())
       {
          return true;
       }
       else
       {
-         //we only need to syncronize if the buffer is empty
+         // we only need to syncronize if the buffer is empty
          synchronized (this)
          {
-            if(!buffer.isEmpty())
+            if (!buffer.isEmpty())
             {
                return true;
             }
-            if(messagesWaiting)
+            if (messagesWaiting)
             {
                wait(timeOut);
             }
@@ -303,17 +302,16 @@
 
    public void stop() throws MessagingException
    {
-      if(!isBrowser)
+      if (!isBrowser)
       {
-          throw new MessagingException(MessagingException.ILLEGAL_STATE,
-                                      "Cannot stop Consumer in non browser mode");
+         throw new MessagingException(MessagingException.ILLEGAL_STATE, "Cannot stop Consumer in non browser mode");
       }
       synchronized (this)
       {
-         //if there are still messages in transit tell the server to stop and wait
-         if(messagesWaiting)
+         // if there are still messages in transit tell the server to stop and wait
+         if (messagesWaiting)
          {
-            //tell the server to stop
+            // tell the server to stop
             channel.send(new SessionConsumerStopMessage(id));
             do
             {
@@ -326,7 +324,7 @@
                   throw new IllegalStateException(e.getMessage());
                }
             }
-            while(messagesWaiting);
+            while (messagesWaiting);
          }
          buffer.clear();
       }
@@ -334,10 +332,9 @@
 
    public void start() throws MessagingException
    {
-      if(!isBrowser)
+      if (!isBrowser)
       {
-          throw new MessagingException(MessagingException.ILLEGAL_STATE,
-                                      "Cannot stop Consumer in non browser mode");
+         throw new MessagingException(MessagingException.ILLEGAL_STATE, "Cannot stop Consumer in non browser mode");
       }
       messagesWaiting = true;
       channel.send(new SessionConsumerStartMessage(id));
@@ -356,7 +353,7 @@
    {
       return id;
    }
-   
+
    public synchronized void handleMessage(final ClientMessage message) throws Exception
    {
       if (closed)
@@ -365,13 +362,6 @@
          return;
       }
 
-      if (cleared)
-      {
-         // Ignore - the session is rolling back and these are inflight
-         // messages
-         return;         
-      }
-            
       message.onReceipt(session, id);
 
       if (handler != null)
@@ -390,7 +380,7 @@
             }
             else
             {
-               session.processed(id, message.getMessageID());
+               session.acknowledge(id, message.getMessageID());
             }
          }
          else
@@ -413,15 +403,10 @@
 
    public synchronized void clear()
    {
-      cleared = true;
-      
+      waitForOnMessageToComplete();
+
       buffer.clear();
    }
-   
-   public synchronized void resume()
-   {
-      cleared = false;
-   }
 
    public int getClientWindowSize()
    {
@@ -502,7 +487,7 @@
          throw new MessagingException(MessagingException.OBJECT_CLOSED, "Consumer is closed");
       }
    }
-   
+
    private void callOnMessage()
    {
       try
@@ -526,18 +511,18 @@
          if (message != null)
          {
             boolean expired = message.isExpired();
-           
+
             flowControl(message.getEncodeSize());
 
             if (!expired)
             {
                onMessageThread = Thread.currentThread();
-               
+
                handler.onMessage(message);
             }
             else
             {
-               session.processed(id, message.getMessageID());
+               session.acknowledge(id, message.getMessageID());
             }
          }
       }
@@ -553,31 +538,35 @@
 
    private void doCleanUp(final boolean sendCloseMessage) throws MessagingException
    {
-      if (closed)
-      {
-         return;
-      }
-
       try
       {
-         // Now we wait for any current handler runners to run.
-         waitForOnMessageToComplete();
+         if (closed)
+         {
+            return;
+         }
 
          closed = true;
 
-         if (receiverThread != null)
+         // Now we wait for any current handler runners to run.
+         waitForOnMessageToComplete();
+
+         synchronized (this)
          {
-            synchronized (this)
+            if (receiverThread != null)
             {
+               // Wake up any receive() thread that might be waiting
+               notify();
+
                messagesWaiting = false;
+               
                // Wake up any receive() thread that might be waiting
                notify();
             }
-         }
 
-         handler = null;
+            handler = null;
 
-         receiverThread = null;
+            receiverThread = null;
+         }
 
          if (sendCloseMessage)
          {

Modified: trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerInternal.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerInternal.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerInternal.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -41,8 +41,6 @@
 
    void clear();
 
-   void resume();
-
    int getClientWindowSize();
 
    int getBufferSize();

Modified: trunk/src/main/org/jboss/messaging/core/client/impl/ClientMessageImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/client/impl/ClientMessageImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/client/impl/ClientMessageImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -102,7 +102,7 @@
    {
       if (session != null)
       {
-         session.processed(consumerID, messageID);
+         session.acknowledge(consumerID, messageID);
       }
    }
 }

Modified: trunk/src/main/org/jboss/messaging/core/client/impl/ClientProducerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/client/impl/ClientProducerImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/client/impl/ClientProducerImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -180,17 +180,17 @@
          channel.send(message);
       }      
       
-      //We only flow control with non-anonymous producers
-      if (address == null && creditFlowControl)
-      {
-         try
-         {
-            availableCredits.acquire(message.getClientMessage().getEncodeSize());
-         }
-         catch (InterruptedException e)
-         {           
-         }         
-      }
+//      //We only flow control with non-anonymous producers
+//      if (address == null && creditFlowControl)
+//      {
+//         try
+//         {
+//            availableCredits.acquire(message.getClientMessage().getEncodeSize());
+//         }
+//         catch (InterruptedException e)
+//         {           
+//         }         
+//      }
    }
 
    public void registerAcknowledgementHandler(final AcknowledgementHandler handler)
@@ -333,17 +333,17 @@
          channel.send(message);
       }
 
-      // We only flow control with non-anonymous producers
-      if (address == null && creditFlowControl)
-      {
-         try
-         {
-            availableCredits.acquire(message.getClientMessage().getEncodeSize());
-         }
-         catch (InterruptedException e)
-         {
-         }
-      }
+//      // We only flow control with non-anonymous producers
+//      if (address == null && creditFlowControl)
+//      {
+//         try
+//         {
+//            availableCredits.acquire(message.getClientMessage().getEncodeSize());
+//         }
+//         catch (InterruptedException e)
+//         {
+//         }
+//      }
    }
 
    private void checkClosed() throws MessagingException

Modified: trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionFactoryImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionFactoryImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionFactoryImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -22,7 +22,6 @@
 import org.jboss.messaging.core.remoting.Channel;
 import org.jboss.messaging.core.remoting.ChannelHandler;
 import org.jboss.messaging.core.remoting.ConnectionRegistry;
-import org.jboss.messaging.core.remoting.FailureListener;
 import org.jboss.messaging.core.remoting.Packet;
 import org.jboss.messaging.core.remoting.RemotingConnection;
 import org.jboss.messaging.core.remoting.impl.ConnectionRegistryImpl;
@@ -41,7 +40,7 @@
  * @author <a href="mailto:ataylor at redhat.com">Andy Taylor</a>
  * @version <tt>$Revision: 3602 $</tt>
  */
-public class ClientSessionFactoryImpl implements ClientSessionFactoryInternal, FailureListener
+public class ClientSessionFactoryImpl implements ClientSessionFactoryInternal
 {
    // Constants
    // ------------------------------------------------------------------------------------
@@ -50,7 +49,7 @@
 
    private static final Logger log = Logger.getLogger(ClientSessionFactoryImpl.class);
 
-   public static final long DEFAULT_PING_PERIOD = 2000;
+   public static final long DEFAULT_PING_PERIOD = 5000;
 
    public static final int DEFAULT_CONSUMER_WINDOW_SIZE = 1024 * 1024;
 
@@ -398,6 +397,20 @@
    {
       sessions.remove(session);
    }
+   
+   public boolean checkFailover(final MessagingException me)
+   {
+      if (backupConnectorFactory != null)
+      {
+         handleFailover(me);
+         
+         return true;
+      }
+      else
+      {
+         return false;
+      }
+   }
 
    // Public
    // ---------------------------------------------------------------------------------------
@@ -468,11 +481,6 @@
       {
          connection = connectionRegistry.getConnection(connectorFactory, transportParams, pingPeriod, callTimeout);
 
-         if (backupConnectorFactory != null)
-         {
-            connection.addFailureListener(this);
-         }
-
          String name = UUIDGenerator.getInstance().generateSimpleStringUUID().toString();
 
          long sessionChannelID = connection.generateChannelID();
@@ -490,7 +498,7 @@
                                                    autoCommitSends,
                                                    autoCommitAcks);
 
-         Channel channel1 = connection.getChannel(1, false, -1, true);
+         Channel channel1 = connection.getChannel(1, -1, true);
 
          try
          {
@@ -517,8 +525,7 @@
 
          int packetConfirmationBatchSize = response.getPacketConfirmationBatchSize();
          
-         Channel sessionChannel = connection.getChannel(sessionChannelID,
-                                                        false,
+         Channel sessionChannel = connection.getChannel(sessionChannelID,                                               
                                                         packetConfirmationBatchSize,                                               
                                                         !hasBackup);
 
@@ -587,9 +594,4 @@
       }
    }
 
-   public void connectionFailed(final MessagingException me)
-   {
-      handleFailover(me);
-   }
-
 }

Modified: trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionFactoryInternal.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionFactoryInternal.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionFactoryInternal.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -22,6 +22,7 @@
 package org.jboss.messaging.core.client.impl;
 
 import org.jboss.messaging.core.client.ClientSessionFactory;
+import org.jboss.messaging.core.exception.MessagingException;
 
 /**
  * A ClientSessionFactoryInternal
@@ -32,4 +33,6 @@
 public interface ClientSessionFactoryInternal extends ClientSessionFactory
 {
    void removeSession(ClientSessionInternal session);
+   
+   boolean checkFailover(MessagingException me);
 }

Modified: trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -32,11 +32,11 @@
 import org.jboss.messaging.core.remoting.FailureListener;
 import org.jboss.messaging.core.remoting.Packet;
 import org.jboss.messaging.core.remoting.RemotingConnection;
-import org.jboss.messaging.core.remoting.ResponseNotifier;
 import org.jboss.messaging.core.remoting.impl.ConnectionRegistryImpl;
 import org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl;
 import org.jboss.messaging.core.remoting.impl.wireformat.ReattachSessionMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.ReattachSessionResponseMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.SessionAcknowledgeMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionAddDestinationMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionBindingQueryMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionBindingQueryResponseMessage;
@@ -49,7 +49,6 @@
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionCreateQueueMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionDeleteQueueMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionFailoverCompleteMessage;
-import org.jboss.messaging.core.remoting.impl.wireformat.SessionProcessedMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionQueueQueryMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionQueueQueryResponseMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionRemoveDestinationMessage;
@@ -103,7 +102,7 @@
  * $Id: ClientSessionImpl.java 3603 2008-01-21 18:49:20Z timfox $
  * 
  */
-public class ClientSessionImpl implements ClientSessionInternal
+public class ClientSessionImpl implements ClientSessionInternal, FailureListener
 {
    // Constants ----------------------------------------------------------------------------
 
@@ -157,6 +156,10 @@
    private boolean forceNotSameRM;
 
    private final IDGenerator idGenerator = new SimpleIDGenerator(0);
+   
+   private volatile boolean failedOver;
+   
+   private volatile boolean started;
 
    // Constructors ----------------------------------------------------------------------------
 
@@ -209,6 +212,8 @@
       this.version = version;
 
       connectionRegistry = ConnectionRegistryImpl.instance;
+      
+      remotingConnection.addFailureListener(this);
    }
 
    // ClientSession implementation
@@ -221,7 +226,7 @@
                            final boolean temp) throws MessagingException
    {
       checkClosed();
-
+      
       SessionCreateQueueMessage request = new SessionCreateQueueMessage(address, queueName, filterString, durable, temp);
 
       channel.sendBlocking(request);
@@ -468,25 +473,31 @@
    public void rollback() throws MessagingException
    {
       checkClosed();
-
+      
+      //We do a "JMS style" rollback where the session is stopped, and the buffer is cancelled back
+      //first before rolling back
+      //This ensures messages are received in the same order after rollback w.r.t. to messages in the buffer
+      //For core we could just do a straight rollback, it really depends if we want JMS style semantics or not...
+      
+      boolean wasStarted = started;
+      
+      if (wasStarted)
+      {
+         stop();
+      }
+      
       // We need to make sure we don't get any inflight messages
       for (ClientConsumerInternal consumer : consumers.values())
       {
          consumer.clear();
       }
 
-      channel.sendBlocking(new PacketImpl(PacketImpl.SESS_ROLLBACK), new ResponseNotifier()
+      channel.sendBlocking(new PacketImpl(PacketImpl.SESS_ROLLBACK));
+      
+      if (wasStarted)
       {
-         public void onResponseReceived()
-         {
-            // This needs to be called on before the blocking thread is awoken
-            // hence the ResponseNotifier
-            for (ClientConsumerInternal consumer : consumers.values())
-            {
-               consumer.resume();
-            }
-         }
-      });
+         start();
+      }
    }
 
    public ClientMessage createClientMessage(final byte type,
@@ -547,15 +558,25 @@
    public void start() throws MessagingException
    {
       checkClosed();
-
-      channel.send(new PacketImpl(PacketImpl.SESS_START));
+      
+      if (!started)
+      {           
+         channel.send(new PacketImpl(PacketImpl.SESS_START));
+         
+         started = true;
+      }
    }
 
    public void stop() throws MessagingException
    {
       checkClosed();
-
-      channel.sendBlocking(new PacketImpl(PacketImpl.SESS_STOP));
+      
+      if (started)
+      {
+         channel.sendBlocking(new PacketImpl(PacketImpl.SESS_STOP));
+         
+         started = false;
+      }
    }
 
    public void addFailureListener(final FailureListener listener)
@@ -581,12 +602,13 @@
       return name;
    }
 
-   public void processed(final long consumerID, final long messageID) throws MessagingException
+   // This acknowledges all messages received by the consumer so far
+   public void acknowledge(final long consumerID, final long messageID) throws MessagingException
    {
       checkClosed();
 
-      SessionProcessedMessage message = new SessionProcessedMessage(consumerID, messageID, blockOnAcknowledge);
-
+      SessionAcknowledgeMessage message = new SessionAcknowledgeMessage(consumerID, messageID, blockOnAcknowledge);
+  
       if (blockOnAcknowledge)
       {
          channel.sendBlocking(message);
@@ -688,8 +710,6 @@
       }
 
       doCleanup();
-
-      sessionFactory.removeSession(this);
    }
    
    public synchronized void cleanUp() throws Exception
@@ -702,8 +722,6 @@
       cleanUpChildren();
       
       doCleanup();      
-      
-      sessionFactory.removeSession(this);
    }
    
    //Needs to be synchronized to prevent issues with occurring concurrently with close()
@@ -728,7 +746,7 @@
 
          Packet request = new ReattachSessionMessage(name, channel.getLastReceivedCommandID());
 
-         Channel channel1 = backupConnection.getChannel(1, false, -1, true);
+         Channel channel1 = backupConnection.getChannel(1, -1, true);
 
          ReattachSessionResponseMessage response = (ReattachSessionResponseMessage)channel1.sendBlocking(request);
 
@@ -756,6 +774,8 @@
       
       channel.send(new SessionFailoverCompleteMessage(name));
       
+      failedOver = true;
+      
       return true;
    }
 
@@ -952,23 +972,7 @@
 
       try
       {
-         SessionXAResponseMessage response = (SessionXAResponseMessage)channel.sendBlocking(packet,
-                                                                                            new ResponseNotifier()
-                                                                                            {
-                                                                                               public void onResponseReceived()
-                                                                                               {
-                                                                                                  // This needs to be
-                                                                                                  // called on before
-                                                                                                  // the blocking
-                                                                                                  // thread is awoken
-                                                                                                  // hence the
-                                                                                                  // ResponseNotifier
-                                                                                                  for (ClientConsumerInternal consumer : consumers.values())
-                                                                                                  {
-                                                                                                     consumer.resume();
-                                                                                                  }
-                                                                                               }
-                                                                                            });
+         SessionXAResponseMessage response = (SessionXAResponseMessage)channel.sendBlocking(packet);
 
          if (response.isError())
          {
@@ -1039,6 +1043,26 @@
          throw new XAException(XAException.XAER_RMERR);
       }
    }
+   
+   // FailureListener implementation --------------------------------------------
+   
+   public void connectionFailed(final MessagingException me)
+   {
+      if (!sessionFactory.checkFailover(me))
+      {
+         if (!failedOver)
+         {
+            try
+            {
+               cleanUp();
+            }
+            catch (Exception e)
+            {
+               log.error("Failed to cleanup session");
+            }
+         }
+      }
+   }
 
    // Public
    // ----------------------------------------------------------------------------
@@ -1091,7 +1115,9 @@
          producerCache.clear();
       }
 
-      channel.close(false);
+      channel.close();
+      
+      remotingConnection.removeFailureListener(this);
 
       synchronized (this)
       {
@@ -1099,6 +1125,8 @@
 
          connectionRegistry.returnConnection(remotingConnection.getID());
       }
+      
+      sessionFactory.removeSession(this);
    }
    
    private void cleanUpChildren() throws Exception

Modified: trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionInternal.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionInternal.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/client/impl/ClientSessionInternal.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -31,7 +31,7 @@
 {
    String getName();
 
-   void processed(long consumerID, long messageID) throws MessagingException;
+   void acknowledge(long consumerID, long messageID) throws MessagingException;
 
    void addConsumer(ClientConsumerInternal consumer);
 

Modified: trunk/src/main/org/jboss/messaging/core/postoffice/PostOffice.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/postoffice/PostOffice.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/postoffice/PostOffice.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -28,7 +28,6 @@
 
 import org.jboss.messaging.core.filter.Filter;
 import org.jboss.messaging.core.paging.PagingManager;
-import org.jboss.messaging.core.paging.impl.PageMessageImpl;
 import org.jboss.messaging.core.server.MessageReference;
 import org.jboss.messaging.core.server.MessagingComponent;
 import org.jboss.messaging.core.server.ServerMessage;

Modified: trunk/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -151,7 +151,7 @@
 
    public boolean addDestination(final SimpleString address, final boolean durable) throws Exception
    {
-      boolean added = addressManager.addDestination(address);// destinations.addIfAbsent(address);
+      boolean added = addressManager.addDestination(address);
 
       if (added)
       {
@@ -268,7 +268,7 @@
          }
 
          List<Binding> bindings = addressManager.getBindings(address);
-
+         
          List<MessageReference> refs = new ArrayList<MessageReference>();
 
          if (bindings != null)
@@ -406,5 +406,4 @@
          }
       }
    }
-
 }

Modified: trunk/src/main/org/jboss/messaging/core/postoffice/impl/SimpleAddressManager.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/postoffice/impl/SimpleAddressManager.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/postoffice/impl/SimpleAddressManager.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -69,6 +69,7 @@
       bindings.add(binding);
       return prevBindings != null;
    }
+   
    public List<Binding> getBindings(final SimpleString address)
    {
       return mappings.get(address);
@@ -116,7 +117,6 @@
       return mappings;
    }
 
-
    public Binding removeBinding(final SimpleString queueName)
    {
       Binding binding = nameMap.remove(queueName);
@@ -131,6 +131,7 @@
    public boolean removeMapping(final SimpleString address, final SimpleString queueName)
    {
       List<Binding> bindings = mappings.get(address);
+      
       Binding binding = removeMapping(queueName, bindings);
 
       if(bindings.isEmpty())
@@ -143,6 +144,7 @@
    protected Binding removeMapping(final SimpleString queueName, final List<Binding> bindings)
    {
       Binding binding = null;
+      
       for (Iterator<Binding> iter = bindings.iterator(); iter.hasNext();)
       {
          Binding b = iter.next();
@@ -161,7 +163,7 @@
       }
 
       bindings.remove(binding);
+      
       return binding;
    }
-
 }

Modified: trunk/src/main/org/jboss/messaging/core/postoffice/impl/WildcardAddressManager.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/postoffice/impl/WildcardAddressManager.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/postoffice/impl/WildcardAddressManager.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -39,10 +39,15 @@
 public class WildcardAddressManager extends SimpleAddressManager
 {
    static final char SINGLE_WORD = '*';
+
    static final char ANY_WORDS = '#';
+
    static final char DELIM = '.';
+
    static final SimpleString SINGLE_WORD_SIMPLESTRING = new SimpleString("*");
+
    static final SimpleString ANY_WORDS_SIMPLESTRING = new SimpleString("#");
+
    /**
     * This is the actual wild card binding, for every binding added here 1 or more actual bindings will be added.
     * i.e. A binding for A.* will bind the same queue to A.B and A.C if they are its linked addresses
@@ -70,14 +75,14 @@
    public boolean addMapping(final SimpleString address, final Binding binding)
    {
       Address add = wildcardDestinations.get(address);
-      //if this isnt a wildcard destination then just add normally
+      // if this isnt a wildcard destination then just add normally
       if (add == null)
       {
          return super.addMapping(address, binding);
       }
       else
       {
-         //add this as a wildcard binding and add a new binding to any linked addresses.
+         // add this as a wildcard binding and add a new binding to any linked addresses.
          for (Address destination : add.getLinkedAddresses())
          {
             BindingImpl binding1 = new BindingImpl(destination.getAddress(), binding.getQueue());
@@ -107,7 +112,7 @@
    public boolean removeMapping(final SimpleString address, final SimpleString queueName)
    {
       Address add = wildcardDestinations.get(address);
-      //if this isnt a wildcard binding just remove normally
+      // if this isnt a wildcard binding just remove normally
       if (add == null)
       {
          return super.removeMapping(address, queueName);
@@ -146,14 +151,14 @@
    public boolean addDestination(final SimpleString address)
    {
       boolean added = super.addDestination(address);
-      //if this is a new destination we compute any wilcard addresses that would match and add if necessary
+      // if this is a new destination we compute any wilcard addresses that would match and add if necessary
       synchronized (actualDestinations)
       {
          if (added)
          {
             Address add = new AddressImpl(address);
             Address prevAddress = actualDestinations.putIfAbsent(address, add);
-            if(prevAddress != null)
+            if (prevAddress != null)
             {
                add = prevAddress;
             }
@@ -239,7 +244,7 @@
       }
       if (pos + 1 < address.getAddressParts().length)
       {
-         return addPart(newAddresses, address, pos+1);
+         return addPart(newAddresses, address, pos + 1);
       }
       else
       {
@@ -259,9 +264,9 @@
          for (int i = 0; i < address.getAddressParts().length; i++)
          {
             SimpleString current = address.getAddressParts()[i];
-            if(i < address.getAddressParts().length-1)
+            if (i < address.getAddressParts().length - 1)
             {
-               next = address.getAddressParts()[i+1];
+               next = address.getAddressParts()[i + 1];
             }
             else
             {
@@ -270,33 +275,33 @@
             if (current.equals(SINGLE_WORD_SIMPLESTRING) && (ANY_WORDS_SIMPLESTRING.equals(next)))
             {
                address.removeAddressPart(i);
-               prev=null;
-               i=-1;
+               prev = null;
+               i = -1;
             }
             else if (current.equals(ANY_WORDS_SIMPLESTRING) && (ANY_WORDS_SIMPLESTRING.equals(next)))
             {
                address.removeAddressPart(i);
-               prev=null;
-               i=-1;
+               prev = null;
+               i = -1;
             }
             else if (current.equals(ANY_WORDS_SIMPLESTRING) && (SINGLE_WORD_SIMPLESTRING.equals(next)))
             {
-               address.removeAddressPart(i+1);
-               prev=null;
-               i=-1;
+               address.removeAddressPart(i + 1);
+               prev = null;
+               i = -1;
             }
             else if (current.equals(ANY_WORDS_SIMPLESTRING) && (ANY_WORDS_SIMPLESTRING.equals(prev)))
             {
-               address.removeAddressPart(i+1);
-               prev=null;
-               i=-1;
+               address.removeAddressPart(i + 1);
+               prev = null;
+               i = -1;
             }
             else
             {
                prev = current;
             }
          }
-         if(!newAddresses.contains(address.getAddress()))
+         if (!newAddresses.contains(address.getAddress()))
          {
             newAddresses.add(address.getAddress());
          }

Modified: trunk/src/main/org/jboss/messaging/core/remoting/Channel.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/Channel.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/Channel.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -11,8 +11,6 @@
  */
 package org.jboss.messaging.core.remoting;
 
-import java.util.concurrent.Executor;
-
 import org.jboss.messaging.core.exception.MessagingException;
 
 /**
@@ -28,15 +26,13 @@
 
    Packet sendBlocking(Packet packet) throws MessagingException;
 
-   Packet sendBlocking(Packet packet, ResponseNotifier notifier) throws MessagingException;
-
-   void replicatePacket(Packet packet, Runnable responseAction);
+   DelayedResult replicatePacket(Packet packet);
    
    void replicateComplete();
 
    void setHandler(ChannelHandler handler);
 
-   void close(boolean onExecutorThread);
+   void close();
 
    void fail();
 
@@ -52,7 +48,5 @@
 
    void unlock();
 
-   Executor getExecutor();
-
    void interruptBlocking();
 }

Modified: trunk/src/main/org/jboss/messaging/core/remoting/Packet.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/Packet.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/Packet.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -49,8 +49,5 @@
    
    boolean isWriteAlways();
    
-   
-//   int getReplicateID();
-//   
-//   void setReplicateID(int id);
+   boolean isRequiresGlobalOrdering();      
 }

Modified: trunk/src/main/org/jboss/messaging/core/remoting/RemotingConnection.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/RemotingConnection.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/RemotingConnection.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -26,7 +26,7 @@
 {
    Object getID();
 
-   Channel getChannel(long channelID, boolean ordered, int packetConfirmationBatchSize, boolean interruptBlockOnFailure);
+   Channel getChannel(long channelID, int packetConfirmationBatchSize, boolean interruptBlockOnFailure);
 
    long generateChannelID();
 

Deleted: trunk/src/main/org/jboss/messaging/core/remoting/ResponseNotifier.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/ResponseNotifier.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/ResponseNotifier.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -1,38 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source
- * Copyright 2005-2008, Red Hat Middleware LLC, and individual contributors
- * by the @authors tag. See the copyright.txt in the distribution for a
- * full listing of individual contributors.
- *
- * This is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License as
- * published by the Free Software Foundation; either version 2.1 of
- * the License, or (at your option) any later version.
- *
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this software; if not, write to the Free
- * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
- */
-
-
-package org.jboss.messaging.core.remoting;
-
-/**
- * A ResponseNotifier
- *
- * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
- * 
- * Created 26 Sep 2008 10:57:15
- *
- *
- */
-public interface ResponseNotifier
-{
-   void onResponseReceived();
-}

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/ConnectionRegistryImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/ConnectionRegistryImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/ConnectionRegistryImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -98,12 +98,12 @@
 
          RemotingConnection connection = new RemotingConnectionImpl(tc,
                                                                     callTimeout,
-                                                                    pingInterval,
-                                                                    null,
+                                                                    pingInterval,                                                             
                                                                     pingExecutor,
                                                                     null,
                                                                     null,
-                                                                    true);        
+                                                                    true,
+                                                                    null);        
 
          handler.conn = connection;
 
@@ -141,12 +141,12 @@
 
       RemotingConnection connection = new RemotingConnectionImpl(tc,
                                                                  callTimeout,
-                                                                 pingInterval,
-                                                                 null,
+                                                                 pingInterval,                                                            
                                                                  pingExecutor,
                                                                  null,
                                                                  null,
-                                                                 true);        
+                                                                 true,
+                                                                 null);        
 
       handler.conn = connection;
 

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/RemotingConnectionImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/RemotingConnectionImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/RemotingConnectionImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -12,20 +12,6 @@
 
 package org.jboss.messaging.core.remoting.impl;
 
-import org.jboss.messaging.core.exception.MessagingException;
-import org.jboss.messaging.core.logging.Logger;
-import org.jboss.messaging.core.remoting.Channel;
-import org.jboss.messaging.core.remoting.ChannelHandler;
-import org.jboss.messaging.core.remoting.FailureListener;
-import org.jboss.messaging.core.remoting.Interceptor;
-import org.jboss.messaging.core.remoting.Packet;
-import org.jboss.messaging.core.remoting.RemotingConnection;
-import org.jboss.messaging.core.remoting.ResponseNotifier;
-import org.jboss.messaging.core.remoting.impl.wireformat.CreateSessionMessage;
-import org.jboss.messaging.core.remoting.impl.wireformat.CreateSessionResponseMessage;
-import org.jboss.messaging.core.remoting.impl.wireformat.MessagingExceptionMessage;
-import org.jboss.messaging.core.remoting.impl.wireformat.NullResponseMessage;
-import org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.CREATESESSION;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.CREATESESSION_RESP;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.EXCEPTION;
@@ -37,6 +23,7 @@
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.REATTACH_SESSION;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.REATTACH_SESSION_RESP;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.REPLICATION_RESPONSE;
+import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_ACKNOWLEDGE;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_ADD_DESTINATION;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_BINDINGQUERY;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_BINDINGQUERY_RESP;
@@ -55,7 +42,6 @@
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_FAILOVER_COMPLETE;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_FLOWTOKEN;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_MANAGEMENT_SEND;
-import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_PROCESSED;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_PRODUCER_CLOSE;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_QUEUEQUERY;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_QUEUEQUERY_RESP;
@@ -84,11 +70,43 @@
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_XA_SET_TIMEOUT_RESP;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_XA_START;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_XA_SUSPEND;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.jboss.messaging.core.exception.MessagingException;
+import org.jboss.messaging.core.logging.Logger;
+import org.jboss.messaging.core.remoting.Channel;
+import org.jboss.messaging.core.remoting.ChannelHandler;
+import org.jboss.messaging.core.remoting.DelayedResult;
+import org.jboss.messaging.core.remoting.FailureListener;
+import org.jboss.messaging.core.remoting.Interceptor;
+import org.jboss.messaging.core.remoting.Packet;
+import org.jboss.messaging.core.remoting.RemotingConnection;
+import org.jboss.messaging.core.remoting.impl.wireformat.CreateSessionMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.CreateSessionResponseMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.MessagingExceptionMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.NullResponseMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl;
 import org.jboss.messaging.core.remoting.impl.wireformat.PacketsConfirmedMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.Ping;
 import org.jboss.messaging.core.remoting.impl.wireformat.Pong;
 import org.jboss.messaging.core.remoting.impl.wireformat.ReattachSessionMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.ReattachSessionResponseMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.SessionAcknowledgeMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionAddDestinationMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionBindingQueryMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionBindingQueryResponseMessage;
@@ -105,7 +123,6 @@
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionDeleteQueueMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionDeliveryCompleteMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionFailoverCompleteMessage;
-import org.jboss.messaging.core.remoting.impl.wireformat.SessionProcessedMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionProducerCloseMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionProducerFlowCreditMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionQueueQueryMessage;
@@ -131,28 +148,8 @@
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionXAStartMessage;
 import org.jboss.messaging.core.remoting.spi.Connection;
 import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
-import org.jboss.messaging.util.ExecutorFactory;
-import org.jboss.messaging.util.Future;
-import org.jboss.messaging.util.OrderedExecutorFactory;
 import org.jboss.messaging.util.SimpleIDGenerator;
 
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.Executor;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
 /**
  * @author <a href="tim.fox at jboss.com">Tim Fox</a>
  * @author <a href="mailto:jmesnil at redhat.com">Jeff Mesnil</a>
@@ -183,8 +180,6 @@
 
    private final long blockingCallTimeout;
 
-   private final ExecutorFactory executorFactory;
-
    private Runnable pinger;
 
    private final List<Interceptor> interceptors;
@@ -219,33 +214,28 @@
    private volatile SimpleIDGenerator idGenerator = new SimpleIDGenerator(10);
 
    private boolean idGeneratorSynced = false;
-
+   
+   private final ReadWriteLock readWriteLock;
+   
+   private final Object transferLock = new Object();
+   
    // Constructors
    // ---------------------------------------------------------------------------------
 
    public RemotingConnectionImpl(final Connection transportConnection,
                                  final long blockingCallTimeout,
-                                 final long pingPeriod,
-                                 final ExecutorService handlerExecutor,
+                                 final long pingPeriod,             
                                  final ScheduledExecutorService pingExecutor,
                                  final List<Interceptor> interceptors,
                                  final RemotingConnection replicatingConnection,
-                                 final boolean active)
+                                 final boolean active,
+                                 final ReadWriteLock readWriteLock)
 
    {
       this.transportConnection = transportConnection;
 
       this.blockingCallTimeout = blockingCallTimeout;
 
-      if (handlerExecutor != null)
-      {
-         executorFactory = new OrderedExecutorFactory(handlerExecutor);
-      }
-      else
-      {
-         executorFactory = null;
-      }
-
       this.interceptors = interceptors;
 
       this.replicatingConnection = replicatingConnection;
@@ -255,9 +245,11 @@
       this.pingPeriod = pingPeriod;
 
       this.pingExecutor = pingExecutor;
+      
+      this.readWriteLock = readWriteLock;
 
       // Channel zero is reserved for pinging
-      pingChannel = getChannel(0, false, -1, false);
+      pingChannel = getChannel(0, -1, false);
 
       final ChannelHandler ppHandler = new PingPongHandler();
 
@@ -288,8 +280,7 @@
       return transportConnection.getID();
    }
 
-   public synchronized Channel getChannel(final long channelID,
-                                          final boolean ordered,
+   public synchronized Channel getChannel(final long channelID,                      
                                           final int packetConfirmationBatchSize,
                                           final boolean interruptBlockOnFailure)
    {
@@ -297,7 +288,7 @@
 
       if (channel == null)
       {
-         channel = new ChannelImpl(this, channelID, ordered, packetConfirmationBatchSize, interruptBlockOnFailure);
+         channel = new ChannelImpl(this, channelID, packetConfirmationBatchSize, interruptBlockOnFailure);
 
          channels.put(channelID, channel);
       }
@@ -416,14 +407,49 @@
       final Packet packet = decode(buffer);
 
       final long channelID = packet.getChannelID();
-
-      synchronized (this)
+      
+      //FIXME - need to redo global ordering since this won't work with multiple connections
+      //Instead use lastSeq technique
+      
+      final boolean useLock = readWriteLock != null;
+      
+      if (useLock)
       {
-         final ChannelImpl channel = channels.get(channelID);
+         if (packet.isRequiresGlobalOrdering() || packet.getType() == REPLICATION_RESPONSE)
+         {
+            readWriteLock.writeLock().lock();
+         }
+         else
+         {
+            readWriteLock.readLock().lock();
+         }
+      }
+      
+      try
+      {
+         // This needs to be synchronized so plays nice with transfer connection
+         synchronized (transferLock)
+         {
+            final ChannelImpl channel = channels.get(channelID);
 
-         if (channel != null)
+            if (channel != null)
+            {
+               channel.handlePacket(packet);
+            }
+         }    
+      }
+      finally
+      {
+         if (useLock)
          {
-            channel.handlePacket(packet);
+            if (packet.isRequiresGlobalOrdering() || packet.getType() == REPLICATION_RESPONSE)
+            {
+               readWriteLock.writeLock().unlock();
+            }
+            else
+            {
+               readWriteLock.readLock().unlock();
+            }
          }
       }
    }
@@ -469,7 +495,7 @@
          future.cancel(false);
       }
 
-      pingChannel.close(false);
+      pingChannel.close();
 
       destroyed = true;
 
@@ -483,10 +509,12 @@
 
       for (Channel channel : channels.values())
       {
-         channel.close(false);
+         channel.close();
       }
    }
 
+   // private static AtomicInteger specialSeq = new AtomicInteger(0);
+
    private void doWrite(final Packet packet)
    {
       final MessagingBuffer buffer = transportConnection.createBuffer(INITIAL_BUFFER_SIZE);
@@ -589,9 +617,9 @@
             packet = new SessionConsumerStartMessage();
             break;
          }
-         case SESS_PROCESSED:
+         case SESS_ACKNOWLEDGE:
          {
-            packet = new SessionProcessedMessage();
+            packet = new SessionAcknowledgeMessage();
             break;
          }
          case SESS_COMMIT:
@@ -808,8 +836,6 @@
    {
       private final long id;
 
-      private final Executor executor;
-
       private ChannelHandler handler;
 
       private Packet response;
@@ -834,8 +860,6 @@
 
       private Thread blockThread;
 
-      private ResponseNotifier responseNotifier;
-
       private final Lock lock = new ReentrantLock();
 
       private final Condition sendCondition = lock.newCondition();
@@ -845,12 +869,11 @@
       private final Object sendLock = new Object();
 
       private boolean failingOver;
-      
-      private final Queue<Runnable> responseActions = new ConcurrentLinkedQueue<Runnable>();
 
+      private final Queue<DelayedResult> responseActions = new ConcurrentLinkedQueue<DelayedResult>();
+
       private ChannelImpl(final RemotingConnectionImpl connection,
                           final long id,
-                          final boolean ordered,
                           final int packetConfirmationBatchSize,
                           final boolean interruptBlockOnFailure)
       {
@@ -858,21 +881,12 @@
 
          this.id = id;
 
-         if (ordered && connection.executorFactory != null)
-         {
-            executor = connection.executorFactory.getExecutor();
-         }
-         else
-         {
-            executor = null;
-         }
-
          if (connection.replicatingConnection != null)
          {
             // Don't want to send confirmations if replicating to backup
             this.packetConfirmationBatchSize = -1;
 
-            replicatingChannel = connection.replicatingConnection.getChannel(id, false, -1, interruptBlockOnFailure);
+            replicatingChannel = connection.replicatingConnection.getChannel(id, -1, interruptBlockOnFailure);
 
             replicatingChannel.setHandler(new ReplicatedPacketsConfirmedChannelHandler());
          }
@@ -927,7 +941,6 @@
       // This must never called by more than one thread concurrently
       public void send(final Packet packet)
       {
-         // Must be protected by lock since on session, deliveries can occur at same time as blocking responses
          synchronized (sendLock)
          {
             packet.setChannelID(id);
@@ -962,20 +975,14 @@
          }
       }
 
-      public Executor getExecutor()
-      {
-         return executor;
-      }
-
       // This must never called by more than one thread concurrently
       public Packet sendBlocking(final Packet packet) throws MessagingException
       {
-         return sendBlocking(packet, null);
-      }
-
-      // This must never called by more than one thread concurrently
-      public Packet sendBlocking(final Packet packet, final ResponseNotifier notifier) throws MessagingException
-      {
+         if (closed)
+         {
+            throw new MessagingException(MessagingException.NOT_CONNECTED, "Connection is destroyed");            
+         }
+         
          packet.setChannelID(id);
 
          lock.lock();
@@ -998,8 +1005,6 @@
 
             blockThread = Thread.currentThread();
 
-            responseNotifier = notifier;
-
             response = null;
 
             connection.doWrite(packet);
@@ -1056,22 +1061,35 @@
             lock.unlock();
          }
       }
-            
-      public void replicatePacket(final Packet packet, final Runnable responseAction)
+
+      /*
+       * With replication we must satisfy two rules:
+       * 1) A command must replicated and processed on the backup and liive before the result on the live is returned
+       * 2) A command must be processed on the live before the next command is processed on the live
+       * We replicate as follows:
+       * As a command arrives on the live, we replicate to the backup where it will get processed, we then immediately process
+       * it on the live, but we stop short of sending the result back from the live until the result of completion has 
+       * arrived back from the backup. This is what the DelayedResult is used for.
+       */
+      public DelayedResult replicatePacket(final Packet packet)
       {
          if (replicatingChannel != null)
          {
             // Must be synchronized since can be called by incoming session commands but also by deliveries
             synchronized (this)
             {
-               responseActions.add(responseAction);
-   
+               DelayedResult result = new DelayedResult();
+               
+               responseActions.add(result);
+
                replicatingChannel.send(packet);
+               
+               return result;
             }
          }
          else
          {
-            responseAction.run();
+            return null;
          }
       }
 
@@ -1088,17 +1106,18 @@
             connection.doWrite(packet);
          }
       }
-
+      
+      // This will never get called concurrently by more than one thread
       public void replicateResponseReceived()
       {
-         Runnable action = responseActions.poll();
-
-         if (action == null)
+         DelayedResult result = responseActions.poll();
+         
+         if (result == null)
          {
             throw new IllegalStateException("Cannot find response action");
          }
 
-         action.run();
+         result.replicated();
       }
 
       public void setHandler(final ChannelHandler handler)
@@ -1106,31 +1125,21 @@
          this.handler = handler;
       }
 
-      public void close(boolean onExecutorThread)
+      public void close()
       {
          if (closed)
          {
             return;
          }
 
-         synchronized (connection)
+         if (!connection.destroyed && connection.channels.remove(id) == null)
          {
-            if (!connection.destroyed && connection.channels.remove(id) == null)
-            {
-               throw new IllegalArgumentException("Cannot find channel with id " + id + " to close");
-            }
-         }
+            throw new IllegalArgumentException("Cannot find channel with id " + id + " to close");
+         }         
 
-         if (!onExecutorThread)
-         {
-            waitForExecutorToComplete();
-         }
-
          if (replicatingChannel != null)
          {
-            replicatingChannel.close(false);
-
-           // replicatingChannel = null;
+            replicatingChannel.close();
          }
 
          closed = true;
@@ -1160,34 +1169,14 @@
          return replicatingChannel;
       }
 
-      private void waitForExecutorToComplete()
-      {
-         if (executor != null)
-         {
-            // Wait for anything in the executor to complete
-            final Future future = new Future();
-
-            executor.execute(future);
-
-            boolean ok = future.await(10000);
-
-            if (!ok)
-            {
-               log.warn("Timed out waiting for executor to complete");
-            }
-         }
-      }
-
       public void transferConnection(final RemotingConnection newConnection)
       {
          // Needs to synchronize on the connection to make sure no packets from
          // the old connection get processed after transfer has occurred
-         synchronized (connection)
+         synchronized (connection.transferLock)
          {
             connection.channels.remove(id);
 
-            waitForExecutorToComplete();
-
             // And switch it
 
             final RemotingConnectionImpl rnewConnection = (RemotingConnectionImpl)newConnection;
@@ -1195,8 +1184,6 @@
             rnewConnection.channels.put(id, this);
 
             connection = rnewConnection;
-
-         //   replicatingChannel = null;
          }
       }
 
@@ -1230,6 +1217,11 @@
          lock.unlock();
       }
 
+      // we need to do a thorough investigation of how packets confirmed get
+      //      
+      // a) replicated from client through live to backup without dealing with on live
+      // b) got redirected back to client from server
+
       private void handlePacket(final Packet packet)
       {
          if (packet.getType() == PACKETS_CONFIRMED)
@@ -1238,27 +1230,7 @@
             {
                final PacketsConfirmedMessage msg = (PacketsConfirmedMessage)packet;
 
-               if (executor == null)
-               {
-                  clearUpTo(msg.getCommandID());
-               }
-               else
-               {
-                  executor.execute(new Runnable()
-                  {
-                     public void run()
-                     {
-                        try
-                        {
-                           clearUpTo(msg.getCommandID());
-                        }
-                        catch (Exception e)
-                        {
-                           log.error("Failed to clear up to", e);
-                        }
-                     }
-                  });
-               }
+               clearUpTo(msg.getCommandID());
             }
             else if (replicatingChannel != null)
             {
@@ -1305,11 +1277,6 @@
 
                try
                {
-                  if (responseNotifier != null)
-                  {
-                     responseNotifier.onResponseReceived();
-                  }
-
                   sendCondition.signal();
                }
                finally
@@ -1318,31 +1285,9 @@
                }
             }
             else if (handler != null)
-            {
-               if (executor == null)
-               {
-                  checkConfirmation(packet);
-                  handler.handlePacket(packet);
-               }
-               else
-               {
-                  executor.execute(new Runnable()
-                  {
-                     public void run()
-                     {
-                        try
-                        {
-                           checkConfirmation(packet);
-
-                           handler.handlePacket(packet);
-                        }
-                        catch (Exception e)
-                        {
-                           log.error("Failed to handle packet", e);
-                        }
-                     }
-                  });
-               }
+            {              
+               checkConfirmation(packet);
+               handler.handlePacket(packet);                               
             }
             else
             {
@@ -1370,6 +1315,8 @@
          }
       }
 
+      // private volatile int lastSentID;
+
       private void addToCache(final Packet packet)
       {
          if (resendCache != null)
@@ -1381,7 +1328,7 @@
       private void clearUpTo(final int lastReceivedCommandID)
       {
          final int numberToClear = 1 + lastReceivedCommandID - firstStoredCommandID;
-
+        
          if (numberToClear == -1)
          {
             throw new IllegalArgumentException("Invalid lastReceivedCommandID: " + lastReceivedCommandID);
@@ -1393,10 +1340,14 @@
 
             if (packet == null)
             {
-               throw new IllegalStateException("Can't find packet to clear: " + " last received command id " +
+               throw new IllegalStateException(System.identityHashCode(this) + 
+                                               " Can't find packet to clear: " +
+                                               " last received command id " +
                                                lastReceivedCommandID +
                                                " first stored command id " +
                                                firstStoredCommandID +
+                                               " cache size " +
+                                               this.resendCache.size() +
                                                " channel id " +
                                                id);
             }

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/RemotingServiceImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/RemotingServiceImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/RemotingServiceImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -20,9 +20,8 @@
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.jboss.messaging.core.config.Configuration;
 import org.jboss.messaging.core.config.TransportConfiguration;
@@ -41,7 +40,6 @@
 import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
 import org.jboss.messaging.core.server.MessagingServer;
 import org.jboss.messaging.core.server.impl.MessagingServerPacketHandler;
-import org.jboss.messaging.util.JBMThreadFactory;
 
 /**
  * @author <a href="mailto:jmesnil at redhat.com">Jeff Mesnil</a>
@@ -65,8 +63,6 @@
 
    private final Set<Acceptor> acceptors = new HashSet<Acceptor>();
 
-   private final ExecutorService remotingExecutor;
-
    private final long callTimeout;
 
    private final Map<Object, RemotingConnection> connections = new ConcurrentHashMap<Object, RemotingConnection>();
@@ -82,6 +78,8 @@
    private volatile boolean backup;
 
    private volatile MessagingServer server;
+   
+   private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(true);
 
    // Static --------------------------------------------------------
 
@@ -89,8 +87,6 @@
 
    public RemotingServiceImpl(final Configuration config)
    {
-      remotingExecutor = Executors.newCachedThreadPool(new JBMThreadFactory("JBM-session-ordering-threads"));
-
       transportConfigs = config.getAcceptorConfigurations();
 
       ClassLoader loader = Thread.currentThread().getContextClassLoader();
@@ -174,20 +170,6 @@
          acceptor.stop();
       }
 
-      remotingExecutor.shutdown();
-
-      try
-      {
-         if (!remotingExecutor.awaitTermination(10000, TimeUnit.MILLISECONDS))
-         {
-            log.warn("Timed out waiting for pool to terminate");
-         }
-      }
-      catch (InterruptedException e)
-      {
-         // Ignore
-      }
-
       started = false;
    }
 
@@ -234,14 +216,14 @@
 
       RemotingConnection rc = new RemotingConnectionImpl(connection,
                                                          callTimeout,
-                                                         -1,
-                                                         remotingExecutor,
+                                                         -1,                            
                                                          null,
                                                          interceptors,
                                                          replicatingConnection,
-                                                         !backup);
+                                                         !backup,
+                                                         readWriteLock);
 
-      Channel channel1 = rc.getChannel(1, false, -1, false);
+      Channel channel1 = rc.getChannel(1,  -1, false);
 
       ChannelHandler handler = new MessagingServerPacketHandler(server, channel1, rc);
 

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/invm/InVMConnection.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/invm/InVMConnection.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/invm/InVMConnection.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -53,9 +53,10 @@
 
    private final String id;
 
-   private volatile boolean started;
+   private boolean closed;
 
-   private static final ExecutorFactory factory = new OrderedExecutorFactory(Executors.newCachedThreadPool(new JBMThreadFactory("JBM-InVM-Transport-Threads")));
+   private static final ExecutorFactory factory =
+      new OrderedExecutorFactory(Executors.newCachedThreadPool(new JBMThreadFactory("JBM-InVM-Transport-Threads")));
 
    private final Executor executor;
 
@@ -75,32 +76,23 @@
       executor = factory.getExecutor();
 
       listener.connectionCreated(this);
-
-      started = true;
    }
 
-   public void close()
+   public synchronized void close()
    {
-      if (!started)
+      if (closed)
       {
          return;
       }
+      
+      //We can't wait for the executor to finish processing, since if the connection is closed on failover on the server
+      //and there are other replication requests still in progress and blocked because of the failover
+      //then it will time out waiting for close.
+      //Instead we let the executor complete after close but ignore the actions
 
-      // Wait for writes to be processed
-      Future future = new Future();
-
-      executor.execute(future);
-
-      boolean ok = future.await(1000);
-
-      if (!ok)
-      {
-         log.warn("Timed out waiting for connection writes to be processed");
-      }
-
       listener.connectionDestroyed(id);
 
-      started = false;
+      closed = true;
    }
 
    public MessagingBuffer createBuffer(final int size)
@@ -121,8 +113,15 @@
          {
             try
             {
-               buffer.getInt(); // read and discard
-               handler.bufferReceived(id, buffer);
+               if (!closed)
+               {
+                  buffer.getInt(); // read and discard
+                  handler.bufferReceived(id, buffer);
+               }
+               else
+               {
+                  //Ignore - buffer came in after connection is closed
+               }               
             }
             catch (Exception e)
             {

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/PacketImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/PacketImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/PacketImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -74,7 +74,7 @@
 
    public static final byte SESS_CONSUMER_START = 45;
 
-   public static final byte SESS_PROCESSED = 46;
+   public static final byte SESS_ACKNOWLEDGE = 46;
 
    public static final byte SESS_COMMIT = 47;
 
@@ -186,7 +186,6 @@
       buffer.putInt(0); // The length gets filled in at the end
       buffer.putByte(type);
       buffer.putLong(channelID);
-     // buffer.putInt(replicateID);
 
       encodeBody(buffer);
 
@@ -202,8 +201,6 @@
    {
       channelID = buffer.getLong();
       
-    //  replicateID = buffer.getInt();
-
       decodeBody(buffer);
    }
 
@@ -230,7 +227,7 @@
       return false;
    }
 
-   public boolean isReHandleResponseOnFailure()
+   public boolean isRequiresGlobalOrdering()
    {
       return false;
    }
@@ -266,17 +263,5 @@
    // Private -------------------------------------------------------
 
    // Inner classes -------------------------------------------------
-   
-//   private int replicateID;
-//   
-//   public int getReplicateID()
-//   {
-//      return replicateID;
-//   }
-//   
-//   public void setReplicateID(int id)
-//   {
-//      this.replicateID = id;
-//   }
 
 }

Copied: trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionAcknowledgeMessage.java (from rev 5095, trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionProcessedMessage.java)
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionAcknowledgeMessage.java	                        (rev 0)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionAcknowledgeMessage.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -0,0 +1,108 @@
+/*
+ * JBoss, Home of Professional Open Source Copyright 2005-2008, Red Hat Middleware LLC, and individual contributors by
+ * the @authors tag. See the copyright.txt in the distribution for a full listing of individual contributors. This is
+ * free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version.
+ * This software is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details. You should have received a copy of the GNU Lesser General Public License along with this software; if not,
+ * write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF
+ * site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.core.remoting.impl.wireformat;
+
+import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
+
+/**
+ * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
+ * @version <tt>$Revision$</tt>
+ */
+public class SessionAcknowledgeMessage extends PacketImpl
+{
+   // Constants -----------------------------------------------------
+
+   // Attributes ----------------------------------------------------
+
+   private long consumerID;
+
+   private long messageID;
+
+   private boolean requiresResponse;
+
+   // Static --------------------------------------------------------
+
+   // Constructors --------------------------------------------------
+
+   public SessionAcknowledgeMessage(final long consumerID, final long messageID, final boolean requiresResponse)
+   {
+      super(SESS_ACKNOWLEDGE);
+
+      this.consumerID = consumerID;
+
+      this.messageID = messageID;
+
+      this.requiresResponse = requiresResponse;
+   }
+
+   public SessionAcknowledgeMessage()
+   {
+      super(SESS_ACKNOWLEDGE);
+   }
+
+   // Public --------------------------------------------------------
+
+   public long getConsumerID()
+   {
+      return consumerID;
+   }
+
+   public long getMessageID()
+   {
+      return messageID;
+   }
+
+   public boolean isRequiresResponse()
+   {
+      return requiresResponse;
+   }
+
+   public void encodeBody(final MessagingBuffer buffer)
+   {
+      buffer.putLong(consumerID);
+
+      buffer.putLong(messageID);
+
+      buffer.putBoolean(requiresResponse);
+   }
+
+   public void decodeBody(final MessagingBuffer buffer)
+   {
+      consumerID = buffer.getLong();
+
+      messageID = buffer.getLong();
+
+      requiresResponse = buffer.getBoolean();
+   }
+
+   public boolean equals(Object other)
+   {
+      if (other instanceof SessionAcknowledgeMessage == false)
+      {
+         return false;
+      }
+
+      SessionAcknowledgeMessage r = (SessionAcknowledgeMessage)other;
+
+      return super.equals(other) && this.consumerID == r.consumerID &&
+             this.messageID == r.messageID &&
+             this.requiresResponse == r.requiresResponse;
+   }
+   // Package protected ---------------------------------------------
+
+   // Protected -----------------------------------------------------
+
+   // Private -------------------------------------------------------
+
+   // Inner classes -------------------------------------------------
+}

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionCreateQueueMessage.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionCreateQueueMessage.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionCreateQueueMessage.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -103,6 +103,11 @@
       return temporary;
    }
    
+   public boolean isRequiresGlobalOrdering()
+   {
+      return true;
+   }
+   
    public void encodeBody(final MessagingBuffer buffer)
    {
       buffer.putSimpleString(address);

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionDeleteQueueMessage.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionDeleteQueueMessage.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionDeleteQueueMessage.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -69,6 +69,11 @@
    {
       return queueName;
    }
+      
+   public boolean isRequiresGlobalOrdering()
+   {
+      return true;
+   }
    
    public void encodeBody(final MessagingBuffer buffer)
    {

Deleted: trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionProcessedMessage.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionProcessedMessage.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionProcessedMessage.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -1,108 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source Copyright 2005-2008, Red Hat Middleware LLC, and individual contributors by
- * the @authors tag. See the copyright.txt in the distribution for a full listing of individual contributors. This is
- * free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as
- * published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version.
- * This software is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
- * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
- * details. You should have received a copy of the GNU Lesser General Public License along with this software; if not,
- * write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF
- * site: http://www.fsf.org.
- */
-
-package org.jboss.messaging.core.remoting.impl.wireformat;
-
-import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
-
-/**
- * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
- * @version <tt>$Revision$</tt>
- */
-public class SessionProcessedMessage extends PacketImpl
-{
-   // Constants -----------------------------------------------------
-
-   // Attributes ----------------------------------------------------
-
-   private long consumerID;
-
-   private long messageID;
-
-   private boolean requiresResponse;
-
-   // Static --------------------------------------------------------
-
-   // Constructors --------------------------------------------------
-
-   public SessionProcessedMessage(final long consumerID, final long messageID, final boolean requiresResponse)
-   {
-      super(SESS_PROCESSED);
-
-      this.consumerID = consumerID;
-
-      this.messageID = messageID;
-
-      this.requiresResponse = requiresResponse;
-   }
-
-   public SessionProcessedMessage()
-   {
-      super(SESS_PROCESSED);
-   }
-
-   // Public --------------------------------------------------------
-
-   public long getConsumerID()
-   {
-      return consumerID;
-   }
-
-   public long getMessageID()
-   {
-      return messageID;
-   }
-
-   public boolean isRequiresResponse()
-   {
-      return requiresResponse;
-   }
-
-   public void encodeBody(final MessagingBuffer buffer)
-   {
-      buffer.putLong(consumerID);
-
-      buffer.putLong(messageID);
-
-      buffer.putBoolean(requiresResponse);
-   }
-
-   public void decodeBody(final MessagingBuffer buffer)
-   {
-      consumerID = buffer.getLong();
-
-      messageID = buffer.getLong();
-
-      requiresResponse = buffer.getBoolean();
-   }
-
-   public boolean equals(Object other)
-   {
-      if (other instanceof SessionProcessedMessage == false)
-      {
-         return false;
-      }
-
-      SessionProcessedMessage r = (SessionProcessedMessage)other;
-
-      return super.equals(other) && this.consumerID == r.consumerID &&
-             this.messageID == r.messageID &&
-             this.requiresResponse == r.requiresResponse;
-   }
-   // Package protected ---------------------------------------------
-
-   // Protected -----------------------------------------------------
-
-   // Private -------------------------------------------------------
-
-   // Inner classes -------------------------------------------------
-}

Modified: trunk/src/main/org/jboss/messaging/core/server/ServerConsumer.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/ServerConsumer.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/server/ServerConsumer.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -22,9 +22,12 @@
 
 package org.jboss.messaging.core.server;
 
+
+import java.util.List;
 import java.util.concurrent.Executor;
 
 
+
 /**
  * 
  * A ServerConsumer
@@ -38,9 +41,9 @@
 	long getID();
 	
 	void close() throws Exception;
+
+	List<MessageReference> cancelRefs() throws Exception;
 	
-	void cancelRefs() throws Exception;
-	
 	void setStarted(boolean started) throws Exception;
 	
 	void receiveCredits(int credits) throws Exception;
@@ -51,7 +54,7 @@
 	
 	void failedOver();
 	
-	void deliver(final long messageID) throws Exception;
+	void deliverReplicated(final long messageID) throws Exception;
 
    void deliver(Executor executor);
 

Modified: trunk/src/main/org/jboss/messaging/core/server/ServerSession.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/ServerSession.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/server/ServerSession.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -64,7 +64,7 @@
 
    void sendScheduled(ServerMessage serverMessage, long scheduledDeliveryTime) throws Exception;
 
-   void processed(final long consumerID, final long messageID) throws Exception;
+   void acknowledge(final long consumerID, final long messageID) throws Exception;
 
    void rollback() throws Exception;
 
@@ -137,7 +137,7 @@
 
    void failedOver() throws Exception;
    
-   void handleReplicatedDelivery(long consumerID, long messageID) throws Exception;
+   void handleReplicatedDelivery(long consumerID, long messageID) throws Exception;   
 
    void promptDelivery(ServerConsumer browser);
 

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -12,6 +12,17 @@
 
 package org.jboss.messaging.core.server.impl;
 
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
 import org.jboss.messaging.core.config.Configuration;
 import org.jboss.messaging.core.config.TransportConfiguration;
 import org.jboss.messaging.core.exception.MessagingException;
@@ -46,13 +57,14 @@
 import org.jboss.messaging.core.transaction.ResourceManager;
 import org.jboss.messaging.core.transaction.impl.ResourceManagerImpl;
 import org.jboss.messaging.core.version.Version;
-import org.jboss.messaging.util.*;
+import org.jboss.messaging.util.ExecutorFactory;
+import org.jboss.messaging.util.GroupIdGenerator;
+import org.jboss.messaging.util.JBMThreadFactory;
+import org.jboss.messaging.util.OrderedExecutorFactory;
+import org.jboss.messaging.util.SimpleString;
+import org.jboss.messaging.util.SimpleStringIdGenerator;
+import org.jboss.messaging.util.VersionLoader;
 
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.*;
-
 /**
  * The messaging server implementation
  * 
@@ -106,7 +118,7 @@
    private ConnectorFactory backupConnectorFactory;
 
    private Map<String, Object> backupConnectorParams;
-
+   
    // plugins
 
    private StorageManager storageManager;
@@ -442,8 +454,7 @@
 
       securityStore.authenticate(username, password);
  
-      Channel channel = connection.getChannel(channelID,
-                                              true,
+      Channel channel = connection.getChannel(channelID,                                             
                                               configuration.getPacketConfirmationBatchSize(),                                             
                                               false);
 
@@ -510,7 +521,7 @@
          return null;
       }
    }
-
+   
    public MessagingServerControlMBean getServerManagement()
    {
       return serverManagement;

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerPacketHandler.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerPacketHandler.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerPacketHandler.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -13,17 +13,16 @@
 package org.jboss.messaging.core.server.impl;
 
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.CREATESESSION;
-import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_FAILOVER_COMPLETE;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.REATTACH_SESSION;
 
 import org.jboss.messaging.core.exception.MessagingException;
 import org.jboss.messaging.core.logging.Logger;
 import org.jboss.messaging.core.remoting.Channel;
 import org.jboss.messaging.core.remoting.ChannelHandler;
+import org.jboss.messaging.core.remoting.DelayedResult;
 import org.jboss.messaging.core.remoting.Packet;
 import org.jboss.messaging.core.remoting.RemotingConnection;
 import org.jboss.messaging.core.remoting.impl.wireformat.CreateSessionMessage;
-import org.jboss.messaging.core.remoting.impl.wireformat.SessionFailoverCompleteMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.MessagingExceptionMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.ReattachSessionMessage;
 import org.jboss.messaging.core.server.MessagingServer;
@@ -57,18 +56,9 @@
    }
 
    public void handlePacket(final Packet packet)
-   {            
-      channel1.replicatePacket(packet, new Runnable()
-      {
-         public void run()
-         {
-            doHandle(packet);
-         }
-      });      
-   }
-   
-   private void doHandle(final Packet packet)
-   {
+   {  
+      DelayedResult result = channel1.replicatePacket(packet);
+      
       Packet response = null;
       
       byte type = packet.getType();
@@ -129,7 +119,22 @@
       
       if (response != null)
       {
-         channel1.send(response);
+         if (result == null)
+         {
+            channel1.send(response);
+         }
+         else
+         {
+            final Packet theResponse = response;
+            
+            result.setResultRunner(new Runnable()
+            {
+               public void run()
+               {
+                  channel1.send(theResponse);
+               }
+            });
+         }
       }
       
       channel1.replicateComplete();

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/QueueImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/QueueImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/QueueImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -174,7 +174,7 @@
       while (iter.hasPrevious())
       {
          MessageReference ref = iter.previous();
-
+         
          ServerMessage msg = ref.getMessage();
 
          if (!checkAndSchedule(ref))

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -22,11 +22,19 @@
 
 package org.jboss.messaging.core.server.impl;
 
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
 import org.jboss.messaging.core.filter.Filter;
 import org.jboss.messaging.core.logging.Logger;
 import org.jboss.messaging.core.persistence.StorageManager;
 import org.jboss.messaging.core.postoffice.PostOffice;
 import org.jboss.messaging.core.remoting.Channel;
+import org.jboss.messaging.core.remoting.DelayedResult;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionDeliveryCompleteMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionReceiveMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionReplicateDeliveryMessage;
@@ -38,15 +46,7 @@
 import org.jboss.messaging.core.server.ServerSession;
 import org.jboss.messaging.core.settings.HierarchicalRepository;
 import org.jboss.messaging.core.settings.impl.QueueSettings;
-import org.jboss.messaging.core.transaction.Transaction;
-import org.jboss.messaging.core.transaction.impl.TransactionImpl;
 
-import java.util.Iterator;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executor;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
 /**
  * Concrete implementation of a ClientConsumer. 
  * 
@@ -104,7 +104,7 @@
    private AtomicBoolean waitingToDeliver = new AtomicBoolean(false);
 
    private boolean delivering = false;
-   
+
    // Constructors
    // ---------------------------------------------------------------------------------
 
@@ -149,8 +149,8 @@
       this.postOffice = postOffice;
 
       this.channel = channel;
-      
-      if(!browseOnly)
+
+      if (!browseOnly)
       {
          messageQueue.addConsumer(this);
       }
@@ -167,14 +167,15 @@
    public HandleStatus handle(final MessageReference ref) throws Exception
    {
       if (availableCredits != null && availableCredits.get() <= 0)
-      {       
+      {
          return HandleStatus.BUSY;
       }
-      
+
       final ServerMessage message = ref.getMessage();
 
       if (message.isExpired())
       {
+         // TODO need to replicate expires
          ref.expire(storageManager, postOffice, queueSettingsRepository);
 
          return HandleStatus.HANDLED;
@@ -199,51 +200,67 @@
          {
             availableCredits.addAndGet(-message.getEncodeSize());
          }
-         
+
          final SessionReceiveMessage packet = new SessionReceiveMessage(id, message, ref.getDeliveryCount() + 1);
-         
-         Runnable run = new Runnable()
+
+         DelayedResult result = channel.replicatePacket(new SessionReplicateDeliveryMessage(id, message.getMessageID()));
+
+         deliveringRefs.add(ref);
+
+         if (result == null)
          {
-            public void run()
+            // Not replicated - just send now
+            channel.send(packet);
+         }
+         else
+         {
+            // Send when replicate delivery response comes back
+            result.setResultRunner(new Runnable()
             {
-               deliveringRefs.add(ref);
-                                            
-               channel.send(packet);
-            }
-         };
-         
-         channel.replicatePacket(new SessionReplicateDeliveryMessage(id, message.getMessageID()), run);
-        
+               public void run()
+               {
+                  channel.send(packet);
+               }
+            });
+         }
+
          return HandleStatus.HANDLED;
       }
    }
-   
+
    public void close() throws Exception
-   {     
+   {
       setStarted(false);
-      
+
       messageQueue.removeConsumer(this);
 
       session.removeConsumer(this);
 
-      cancelRefs();
+      LinkedList<MessageReference> refs = cancelRefs();
+
+      if (!refs.isEmpty())
+      {
+         messageQueue.addListFirst(refs);
+      }
    }
 
-   public void cancelRefs() throws Exception
+   public LinkedList<MessageReference> cancelRefs() throws Exception
    {
+      LinkedList<MessageReference> refs = new LinkedList<MessageReference>();
+
       if (!deliveringRefs.isEmpty())
       {
-         Transaction tx = new TransactionImpl(storageManager, postOffice);
-
          for (MessageReference ref : deliveringRefs)
          {
-            tx.addAcknowledgement(ref);
+            refs.add(ref);
+
+            ref.getQueue().referenceCancelled();
          }
 
          deliveringRefs.clear();
+      }
 
-         tx.rollback(queueSettingsRepository);
-      }
+      return refs;
    }
 
    public void setStarted(final boolean started)
@@ -265,11 +282,11 @@
       if (availableCredits != null)
       {
          int previous = availableCredits.getAndAdd(credits);
-         
+
          if (previous <= 0 && previous + credits > 0)
          {
             promptDelivery();
-         }                 
+         }
       }
    }
 
@@ -277,41 +294,51 @@
    {
       return messageQueue;
    }
- 
+
    public MessageReference getReference(final long messageID) throws Exception
-   {     
-      MessageReference ref = deliveringRefs.poll();
-            
-      return ref;      
+   {
+      // Acknowledge acknowledges all refs delivered by the consumer up to and including the one explicitly
+      // acknowledged
+
+      MessageReference ref;
+      do
+      {
+         ref = deliveringRefs.poll();
+
+         if (ref == null)
+         {
+            throw new IllegalStateException("Could not find reference with id " + messageID +
+                                            " backup " +
+                                            messageQueue.isBackup());
+         }
+      }
+      while (ref.getMessage().getMessageID() != messageID);
+
+      return ref;
    }
-   
-   public void deliver(final long messageID) throws Exception
+
+   public void deliverReplicated(final long messageID) throws Exception
    {
+      // It may not be the first in the queue - since there may be multiple producers
+      // sending to the queue
       MessageReference ref = messageQueue.removeReferenceWithID(messageID);
-         
-      if (ref == null)
+
+      HandleStatus handled = this.handle(ref);
+
+      if (handled != HandleStatus.HANDLED)
       {
-         throw new IllegalStateException("Cannot find ref to deliver " + ref);
+         throw new IllegalStateException("Reference was not handled " + ref + " " + handled);
       }
-      
-      HandleStatus status = handle(ref);
-      
-      if (status != HandleStatus.HANDLED)
-      {
-         throw new IllegalStateException("ref " + ref + " was not handled");
-      }
    }
 
    public void failedOver()
    {
-      synchronized (startStopLock)
-      {
-         started = true;
-      }
-
       if (messageQueue.consumerFailedOver())
       {
-         promptDelivery();
+         if (started)
+         {
+            promptDelivery();
+         }
       }
    }
 
@@ -343,7 +370,7 @@
 
    private void promptDelivery()
    {
-      if(browseOnly)
+      if (browseOnly)
       {
          session.promptDelivery(this);
       }
@@ -355,7 +382,7 @@
 
    // Inner classes
    // ------------------------------------------------------------------------
-      private class DeliveryRunner implements Runnable
+   private class DeliveryRunner implements Runnable
    {
       public void run()
       {
@@ -373,8 +400,8 @@
                }
                channel.send(new SessionReceiveMessage(id, ref.getMessage(), 1));
             }
-            //inform the client there are no more messages
-            if(!iterator.hasNext() || !delivering)
+            // inform the client there are no more messages
+            if (!iterator.hasNext() || !delivering)
             {
                channel.send(new SessionDeliveryCompleteMessage(id));
                iterator = null;

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/ServerProducerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/ServerProducerImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/ServerProducerImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -121,9 +121,9 @@
 	{
 	   creditsToSend.addAndGet(-credits);
 	   
-		Packet packet = new SessionProducerFlowCreditMessage(id, credits);
-		
-		channel.send(packet);	
+//		Packet packet = new SessionProducerFlowCreditMessage(id, credits);
+//		
+//		channel.send(packet);	
 	}
 	
 	public void setWaiting(final boolean waiting)

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -12,6 +12,23 @@
 
 package org.jboss.messaging.core.server.impl;
 
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executor;
+
+import javax.management.Notification;
+import javax.management.NotificationListener;
+import javax.transaction.xa.XAException;
+import javax.transaction.xa.XAResource;
+import javax.transaction.xa.Xid;
+
 import org.jboss.messaging.core.client.management.impl.ManagementHelper;
 import org.jboss.messaging.core.exception.MessagingException;
 import org.jboss.messaging.core.filter.Filter;
@@ -25,10 +42,8 @@
 import org.jboss.messaging.core.postoffice.PostOffice;
 import org.jboss.messaging.core.remoting.Channel;
 import org.jboss.messaging.core.remoting.FailureListener;
-import org.jboss.messaging.core.remoting.Packet;
 import org.jboss.messaging.core.remoting.RemotingConnection;
 import org.jboss.messaging.core.remoting.impl.ByteBufferWrapper;
-import org.jboss.messaging.core.remoting.impl.wireformat.NullResponseMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionBindingQueryResponseMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionCreateConsumerResponseMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionCreateProducerResponseMessage;
@@ -54,20 +69,6 @@
 import org.jboss.messaging.util.SimpleString;
 import org.jboss.messaging.util.SimpleStringIdGenerator;
 
-import javax.management.Notification;
-import javax.management.NotificationListener;
-import javax.transaction.xa.XAException;
-import javax.transaction.xa.XAResource;
-import javax.transaction.xa.Xid;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executor;
-
 /*
  * Session implementation 
  * 
@@ -131,8 +132,6 @@
 
    private final IDGenerator idGenerator = new SimpleIDGenerator(0);
 
-   private volatile boolean closed;
-
    private final String name;
 
    private final MessagingServer server;
@@ -246,7 +245,7 @@
 
       started = s;
    }
-   
+
    public void failedOver() throws Exception
    {
       Set<ServerConsumer> consumersClone = new HashSet<ServerConsumer>(consumers.values());
@@ -255,16 +254,12 @@
       {
          consumer.failedOver();
       }
-
-      started = true;
    }
 
    public void close() throws Exception
    {
-      closed = true;
+      rollback(false);
 
-      channel.close(true);
-
       Set<ServerConsumer> consumersClone = new HashSet<ServerConsumer>(consumers.values());
 
       for (ServerConsumer consumer : consumersClone)
@@ -283,8 +278,6 @@
 
       producers.clear();
 
-      rollback(false);
-
       server.removeSession(name);
    }
 
@@ -305,7 +298,7 @@
 
    public void reStartConsumer(long consumerID)
    {
-       consumers.get(consumerID).start();
+      consumers.get(consumerID).start();
    }
 
    public void send(final ServerMessage msg) throws Exception
@@ -326,7 +319,7 @@
 
             for (MessageReference ref : refs)
             {
-               ref.getQueue().addLast(ref);         
+               ref.getQueue().addLast(ref);
             }
          }
       }
@@ -336,7 +329,6 @@
       }
    }
 
-
    public void sendScheduled(final ServerMessage msg, final long scheduledDeliveryTime) throws Exception
    {
       doSecurity(msg);
@@ -354,9 +346,11 @@
 
             for (MessageReference ref : refs)
             {
-               if(ref.getQueue().isDurable())
+               if (ref.getQueue().isDurable())
                {
-                  storageManager.storeMessageReferenceScheduled(ref.getQueue().getPersistenceID(), msg.getMessageID(), scheduledDeliveryTime);
+                  storageManager.storeMessageReferenceScheduled(ref.getQueue().getPersistenceID(),
+                                                                msg.getMessageID(),
+                                                                scheduledDeliveryTime);
                }
                ref.setScheduledDeliveryTime(scheduledDeliveryTime);
                ref.getQueue().addLast(ref);
@@ -369,43 +363,21 @@
       }
    }
 
-
-   public void processed(final long consumerID, final long messageID) throws Exception
+   public void acknowledge(final long consumerID, final long messageID) throws Exception
    {
       MessageReference ref = consumers.get(consumerID).getReference(messageID);
 
-      // Ref = null would imply consumer is already closed so we could ignore it
-      if (ref != null)
+      if (autoCommitAcks)
       {
-         if (ref.getMessage().getMessageID() != messageID)
-         {
-            throw new IllegalStateException("Invalid order " + ref.getMessage().getMessageID());
-         }
-
-         if (autoCommitAcks)
-         {
-            doAck(ref);
-         }
-         else
-         {
-            tx.addAcknowledgement(ref);
-
-            // Del count is not actually updated in storage unless it's
-            // cancelled
-            ref.incrementDeliveryCount();
-         }
+         doAck(ref);
       }
       else
       {
-         if (!closed)
-         {
-            throw new IllegalStateException(System.identityHashCode(this) + " Could not find ref with id " + messageID);
-         }
-         else
-         {
-            // If closed then might not find ref since processed might come in before send and send
-            // didn't come in since closed
-         }
+         tx.addAcknowledgement(ref);
+
+         // Del count is not actually updated in storage unless it's
+         // cancelled
+         ref.incrementDeliveryCount();
       }
    }
 
@@ -414,25 +386,12 @@
       rollback(true);
    }
 
-   private void rollback(final boolean sendResponse) throws Exception
+   private void doRollback(final Transaction theTx) throws Exception
    {
-      if (tx == null)
-      {
-         // Might be null if XA
-
-         tx = new TransactionImpl(storageManager, postOffice);
-      }
-
-      if (sendResponse)
-      {
-         Packet response = new NullResponseMessage();
-         
-         // Need to write the response now - before redeliveries occur
-         channel.send(response);
-      }
-
       boolean wasStarted = started;
 
+      List<MessageReference> toCancel = new ArrayList<MessageReference>();
+
       for (ServerConsumer consumer : consumers.values())
       {
          if (wasStarted)
@@ -440,11 +399,13 @@
             consumer.setStarted(false);
          }
 
-         consumer.cancelRefs();
+         toCancel.addAll(consumer.cancelRefs());
       }
 
-      tx.rollback(queueSettingsRepository);
+      List<MessageReference> rolledBack = theTx.rollback(queueSettingsRepository);
 
+      rolledBack.addAll(toCancel);
+
       if (wasStarted)
       {
          for (ServerConsumer consumer : consumers.values())
@@ -453,6 +414,46 @@
          }
       }
 
+      // Now cancel the refs back to the queue(s), we sort into queues and cancel back atomically to
+      // preserve order
+
+      Map<Queue, LinkedList<MessageReference>> queueMap = new HashMap<Queue, LinkedList<MessageReference>>();
+
+      for (MessageReference ref : rolledBack)
+      {
+         Queue queue = ref.getQueue();
+
+         LinkedList<MessageReference> list = queueMap.get(queue);
+
+         if (list == null)
+         {
+            list = new LinkedList<MessageReference>();
+
+            queueMap.put(queue, list);
+         }
+
+         list.add(ref);
+      }
+
+      for (Map.Entry<Queue, LinkedList<MessageReference>> entry : queueMap.entrySet())
+      {
+         LinkedList<MessageReference> refs = entry.getValue();
+
+         entry.getKey().addListFirst(refs);
+      }
+   }
+
+   private void rollback(final boolean sendResponse) throws Exception
+   {
+      if (tx == null)
+      {
+         // Might be null if XA
+
+         tx = new TransactionImpl(storageManager, postOffice);
+      }
+
+      doRollback(tx);
+
       tx = new TransactionImpl(storageManager, postOffice);
    }
 
@@ -477,7 +478,7 @@
          return new SessionXAResponseMessage(true, XAException.XAER_PROTO, msg);
       }
 
-      Transaction theTx = resourceManager.getTransaction(xid);
+      Transaction theTx = resourceManager.removeTransaction(xid);
 
       if (theTx == null)
       {
@@ -488,6 +489,9 @@
 
       if (theTx.getState() == Transaction.State.SUSPENDED)
       {
+         // Put it back
+         resourceManager.putTransaction(xid, tx);
+
          return new SessionXAResponseMessage(true,
                                              XAException.XAER_PROTO,
                                              "Cannot commit transaction, it is suspended " + xid);
@@ -495,15 +499,6 @@
 
       theTx.commit();
 
-      boolean removed = resourceManager.removeTransaction(xid);
-
-      if (!removed)
-      {
-         final String msg = "Failed to remove transaction: " + xid;
-
-         return new SessionXAResponseMessage(true, XAException.XAER_PROTO, msg);
-      }
-
       return new SessionXAResponseMessage(false, XAResource.XA_OK, null);
    }
 
@@ -586,7 +581,7 @@
          return new SessionXAResponseMessage(true, XAException.XAER_PROTO, msg);
       }
 
-      Transaction theTx = resourceManager.getTransaction(xid);
+      Transaction theTx = resourceManager.removeTransaction(xid);
 
       if (theTx == null)
       {
@@ -597,6 +592,10 @@
 
       if (theTx.getState() == Transaction.State.SUSPENDED)
       {
+         // Put it back
+
+         resourceManager.putTransaction(xid, tx);
+
          return new SessionXAResponseMessage(true,
                                              XAException.XAER_PROTO,
                                              "Cannot prepare transaction, it is suspended " + xid);
@@ -604,17 +603,6 @@
 
       if (theTx.isEmpty())
       {
-         // Nothing to do - remove it
-
-         boolean removed = resourceManager.removeTransaction(xid);
-
-         if (!removed)
-         {
-            final String msg = "Failed to remove transaction: " + xid;
-
-            return new SessionXAResponseMessage(true, XAException.XAER_PROTO, msg);
-         }
-
          return new SessionXAResponseMessage(false, XAResource.XA_RDONLY, null);
       }
       else
@@ -666,7 +654,7 @@
          return new SessionXAResponseMessage(true, XAException.XAER_PROTO, msg);
       }
 
-      Transaction theTx = resourceManager.getTransaction(xid);
+      Transaction theTx = resourceManager.removeTransaction(xid);
 
       if (theTx == null)
       {
@@ -677,42 +665,16 @@
 
       if (theTx.getState() == Transaction.State.SUSPENDED)
       {
+         // Put it back
+         resourceManager.putTransaction(xid, tx);
+
          return new SessionXAResponseMessage(true,
                                              XAException.XAER_PROTO,
                                              "Cannot rollback transaction, it is suspended " + xid);
       }
 
-      boolean wasStarted = started;
+      doRollback(theTx);
 
-      for (ServerConsumer consumer : consumers.values())
-      {
-         if (wasStarted)
-         {
-            consumer.setStarted(false);
-         }
-
-         consumer.cancelRefs();
-      }
-
-      theTx.rollback(queueSettingsRepository);
-
-      if (wasStarted)
-      {
-         for (ServerConsumer consumer : consumers.values())
-         {
-            consumer.setStarted(true);
-         }
-      }
-
-      boolean removed = resourceManager.removeTransaction(xid);
-
-      if (!removed)
-      {
-         final String msg = "Failed to remove transaction: " + xid;
-
-         return new SessionXAResponseMessage(true, XAException.XAER_PROTO, msg);
-      }
-
       return new SessionXAResponseMessage(false, XAResource.XA_OK, null);
    }
 
@@ -832,7 +794,6 @@
       {
          securityStore.check(address, CheckType.CREATE, this);
       }
-
       Binding binding = postOffice.getBinding(queueName);
 
       if (binding != null)
@@ -902,7 +863,7 @@
                                                               final SimpleString filterString,
                                                               int windowSize,
                                                               int maxRate,
-                                                              boolean isBrowser) throws Exception
+                                                              final boolean isBrowser) throws Exception
    {
       Binding binding = postOffice.getBinding(queueName);
 
@@ -943,7 +904,8 @@
                                                        storageManager,
                                                        queueSettingsRepository,
                                                        postOffice,
-                                                       channel, isBrowser);
+                                                       channel,
+                                                       isBrowser);
 
       SessionCreateConsumerResponseMessage response = new SessionCreateConsumerResponseMessage(windowSize);
 
@@ -1080,9 +1042,11 @@
       producers.get(producerID).send(message);
    }
 
-   public void sendScheduledProducerMessage(final long producerID, final ServerMessage message, final long scheduledDeliveryTime) throws Exception
+   public void sendScheduledProducerMessage(final long producerID,
+                                            final ServerMessage message,
+                                            final long scheduledDeliveryTime) throws Exception
    {
-       producers.get(producerID).sendScheduled(message, scheduledDeliveryTime);  
+      producers.get(producerID).sendScheduled(message, scheduledDeliveryTime);
    }
 
    public int transferConnection(final RemotingConnection newConnection, final int lastReceivedCommandID)
@@ -1099,7 +1063,7 @@
       remotingConnection = newConnection;
 
       remotingConnection.addFailureListener(this);
-      
+
       int serverLastReceivedCommandID = channel.getLastReceivedCommandID();
 
       channel.replayCommands(lastReceivedCommandID);
@@ -1143,10 +1107,15 @@
 
       send(serverMessage);
    }
-   
+
    public void handleReplicatedDelivery(long consumerID, long messageID) throws Exception
    {
-      consumers.get(consumerID).deliver(messageID);
+      ServerConsumer consumer = consumers.get(consumerID);
+
+      if (consumer != null)
+      {
+         consumer.deliverReplicated(messageID);
+      }
    }
 
    // FailureListener implementation
@@ -1168,22 +1137,7 @@
             }
          }
 
-         // We execute this on the session's serial executor, then we can avoid complex synchronization
-         // and ensure no operations are fielded on the session after it is closed
-         channel.getExecutor().execute(new Runnable()
-         {
-            public void run()
-            {
-               try
-               {
-                  close();
-               }
-               catch (Exception e)
-               {
-                  log.error("Failed to close session", e);
-               }
-            }
-         });
+         close();
       }
       catch (Throwable t)
       {
@@ -1250,7 +1204,6 @@
       queue.referenceAcknowledged(ref);
    }
 
-
    private void doSecurity(final ServerMessage msg) throws Exception
    {
       try

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionPacketHandler.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionPacketHandler.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionPacketHandler.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -12,14 +12,8 @@
 
 package org.jboss.messaging.core.server.impl;
 
-import org.jboss.messaging.core.exception.MessagingException;
-import org.jboss.messaging.core.logging.Logger;
-import org.jboss.messaging.core.persistence.StorageManager;
-import org.jboss.messaging.core.remoting.Channel;
-import org.jboss.messaging.core.remoting.ChannelHandler;
-import org.jboss.messaging.core.remoting.Packet;
-import org.jboss.messaging.core.remoting.impl.wireformat.MessagingExceptionMessage;
-import org.jboss.messaging.core.remoting.impl.wireformat.NullResponseMessage;
+
+import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_ACKNOWLEDGE;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_ADD_DESTINATION;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_BINDINGQUERY;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_CLOSE;
@@ -34,7 +28,6 @@
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_FAILOVER_COMPLETE;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_FLOWTOKEN;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_MANAGEMENT_SEND;
-import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_PROCESSED;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_PRODUCER_CLOSE;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_QUEUEQUERY;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_REMOVE_DESTINATION;
@@ -56,6 +49,21 @@
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_XA_SET_TIMEOUT;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_XA_START;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_XA_SUSPEND;
+
+import java.util.List;
+
+import javax.transaction.xa.Xid;
+
+import org.jboss.messaging.core.exception.MessagingException;
+import org.jboss.messaging.core.logging.Logger;
+import org.jboss.messaging.core.persistence.StorageManager;
+import org.jboss.messaging.core.remoting.Channel;
+import org.jboss.messaging.core.remoting.ChannelHandler;
+import org.jboss.messaging.core.remoting.DelayedResult;
+import org.jboss.messaging.core.remoting.Packet;
+import org.jboss.messaging.core.remoting.impl.wireformat.MessagingExceptionMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.NullResponseMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.SessionAcknowledgeMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionAddDestinationMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionBindingQueryMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionConsumerCloseMessage;
@@ -66,7 +74,6 @@
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionCreateProducerMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionCreateQueueMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionDeleteQueueMessage;
-import org.jboss.messaging.core.remoting.impl.wireformat.SessionProcessedMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionProducerCloseMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionQueueQueryMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.SessionRemoveDestinationMessage;
@@ -89,9 +96,6 @@
 import org.jboss.messaging.core.server.ServerMessage;
 import org.jboss.messaging.core.server.ServerSession;
 
-import javax.transaction.xa.Xid;
-import java.util.List;
-
 /**
  * A ServerSessionPacketHandler
  *
@@ -108,7 +112,7 @@
    private final Channel channel;
 
    private final StorageManager storageManager;
-
+   
    public ServerSessionPacketHandler(final ServerSession session,
                                      final Channel channel,
                                      final StorageManager storageManager)
@@ -129,7 +133,7 @@
    public void handlePacket(final Packet packet)
    {
       byte type = packet.getType();
-
+      
       if (type == SESS_SEND || type == SESS_SCHEDULED_SEND)
       {
          SessionSendMessage send = (SessionSendMessage)packet;
@@ -138,30 +142,19 @@
 
          if (msg.getMessageID() == 0L)
          {
-            // must generate message id here, so we know they are in sync
+            // must generate message id here, so we know they are in sync on live and backup
             long id = storageManager.generateUniqueID();
 
             send.getServerMessage().setMessageID(id);
          }
       }
 
-      channel.replicatePacket(packet, new Runnable()
-      {
-         public void run()
-         {
-            doHandle(packet);
-         }
-      });
-   }
-
-   private void doHandle(final Packet packet)
-   {
       Packet response = null;
-
+      
+      DelayedResult result = channel.replicatePacket(packet);
+      
       try
       {
-         byte type = packet.getType();
-
          switch (type)
          {
             case SESS_CREATECONSUMER:
@@ -227,10 +220,10 @@
                response = new NullResponseMessage();
                break;
             }
-            case SESS_PROCESSED:
+            case SESS_ACKNOWLEDGE:
             {
-               SessionProcessedMessage message = (SessionProcessedMessage)packet;
-               session.processed(message.getConsumerID(), message.getMessageID());
+               SessionAcknowledgeMessage message = (SessionAcknowledgeMessage)packet;
+               session.acknowledge(message.getConsumerID(), message.getMessageID());
                if (message.isRequiresResponse())
                {
                   response = new NullResponseMessage();
@@ -246,7 +239,7 @@
             case SESS_ROLLBACK:
             {
                session.rollback();
-               // Rollback response is handled in the rollback() method
+               response = new NullResponseMessage();
                break;
             }
             case SESS_XA_COMMIT:
@@ -281,7 +274,7 @@
             }
             case SESS_XA_ROLLBACK:
             {
-               SessionXARollbackMessage message = (SessionXARollbackMessage)packet;
+               SessionXARollbackMessage message = (SessionXARollbackMessage)packet;              
                response = session.XARollback(message.getXid());
                break;
             }
@@ -436,7 +429,35 @@
 
       if (response != null)
       {
-         channel.send(response);
+         final boolean closeChannel = type == SESS_CLOSE;
+         
+         if (result == null)
+         {
+            //Not clustered - just send now
+            channel.send(response);              
+            
+            if (closeChannel)
+            {
+               channel.close();
+            }
+         }
+         else
+         {
+            final Packet theResponse = response;
+            
+            result.setResultRunner(new Runnable()
+            {
+               public void run()
+               {
+                  channel.send(theResponse);
+                  
+                  if (closeChannel)
+                  {
+                     channel.close();
+                  }
+               }
+            });
+         }
       }
 
       channel.replicateComplete();

Modified: trunk/src/main/org/jboss/messaging/core/transaction/ResourceManager.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/transaction/ResourceManager.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/transaction/ResourceManager.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -39,7 +39,7 @@
    
    Transaction getTransaction(Xid xid);
    
-   boolean removeTransaction(Xid xid);
+   Transaction removeTransaction(Xid xid);
    
    int getTimeoutSeconds();
    

Modified: trunk/src/main/org/jboss/messaging/core/transaction/Transaction.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/transaction/Transaction.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/transaction/Transaction.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -44,12 +44,12 @@
 
    void commit() throws Exception;
 
-   void rollback(HierarchicalRepository<QueueSettings> queueSettingsRepository) throws Exception;
+   List<MessageReference> rollback(HierarchicalRepository<QueueSettings> queueSettingsRepository) throws Exception;
 
    void addMessage(ServerMessage message) throws Exception;
 
    void addAcknowledgement(MessageReference acknowledgement) throws Exception;
-
+   
    int getAcknowledgementsCount();
 
    long getID();

Modified: trunk/src/main/org/jboss/messaging/core/transaction/impl/ResourceManagerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/transaction/impl/ResourceManagerImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/transaction/impl/ResourceManagerImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -65,9 +65,9 @@
       return transactions.putIfAbsent(xid, tx) == null;
    }
 
-   public boolean removeTransaction(final Xid xid)
+   public Transaction removeTransaction(final Xid xid)
    {
-      return transactions.remove(xid) != null;
+      return transactions.remove(xid);
    }
    
    public int getTimeoutSeconds()

Modified: trunk/src/main/org/jboss/messaging/core/transaction/impl/TransactionImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/transaction/impl/TransactionImpl.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/core/transaction/impl/TransactionImpl.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -12,6 +12,15 @@
 
 package org.jboss.messaging.core.transaction.impl;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import javax.transaction.xa.Xid;
+
 import org.jboss.messaging.core.exception.MessagingException;
 import org.jboss.messaging.core.logging.Logger;
 import org.jboss.messaging.core.paging.PageTransactionInfo;
@@ -27,14 +36,6 @@
 import org.jboss.messaging.core.transaction.Transaction;
 import org.jboss.messaging.util.SimpleString;
 
-import javax.transaction.xa.Xid;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
 /**
  * A TransactionImpl
  * 
@@ -195,14 +196,11 @@
 
       ServerMessage message = acknowledgement.getMessage();
 
-      if (message.decrementRefCount() == 0)
+      if (message.decrementRefCount() == 0 && pagingManager != null)
       {
-         if (pagingManager != null)
-         {
-            pagingManager.messageDone(message);
-         }
+         pagingManager.messageDone(message);
       }
-
+      
       if (message.isDurable())
       {
          Queue queue = acknowledgement.getQueue();
@@ -325,7 +323,7 @@
       state = State.COMMITTED;
    }
 
-   public void rollback(final HierarchicalRepository<QueueSettings> queueSettingsRepository) throws Exception
+   public List<MessageReference> rollback(final HierarchicalRepository<QueueSettings> queueSettingsRepository) throws Exception
    {
       if (xid != null)
       {
@@ -352,12 +350,8 @@
          pageTransaction.forget();
       }
 
-      Map<Queue, LinkedList<MessageReference>> queueMap = new HashMap<Queue, LinkedList<MessageReference>>();
-
-      // We sort into lists - one for each queue involved.
-      // Then we cancel back atomicly for each queue adding list on front to
-      // guarantee ordering is preserved
-
+      LinkedList<MessageReference> toCancel = new LinkedList<MessageReference>();
+      
       for (MessageReference ref : acknowledgements)
       {
          Queue queue = ref.getQueue();
@@ -370,31 +364,17 @@
             pagingManager.addSize(message);
          }
 
-         LinkedList<MessageReference> list = queueMap.get(queue);
-
-         if (list == null)
-         {
-            list = new LinkedList<MessageReference>();
-
-            queueMap.put(queue, list);
-         }
-
          if (ref.cancel(storageManager, postOffice, queueSettingsRepository))
          {
-            list.add(ref);
-         }
+            toCancel.add(ref);
+         } 
       }
-
-      for (Map.Entry<Queue, LinkedList<MessageReference>> entry : queueMap.entrySet())
-      {
-         LinkedList<MessageReference> refs = entry.getValue();
-
-         entry.getKey().addListFirst(refs);
-      }
-
+      
       clear();
 
       state = State.ROLLEDBACK;
+      
+      return toCancel;
    }
 
    public int getAcknowledgementsCount()

Modified: trunk/src/main/org/jboss/messaging/util/OrderedExecutorFactory.java
===================================================================
--- trunk/src/main/org/jboss/messaging/util/OrderedExecutorFactory.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/util/OrderedExecutorFactory.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -132,3 +132,5 @@
       }
    }
 }
+
+

Modified: trunk/src/main/org/jboss/messaging/util/VersionLoader.java
===================================================================
--- trunk/src/main/org/jboss/messaging/util/VersionLoader.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/src/main/org/jboss/messaging/util/VersionLoader.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -36,7 +36,7 @@
  */
 public class VersionLoader
 {
-   public static Version load()
+   public static synchronized Version load()
    {
       Properties versionProps = new Properties();
       InputStream in = VersionImpl.class.getClassLoader().getResourceAsStream("version.properties");

Modified: trunk/tests/jms-tests/src/org/jboss/test/messaging/jms/AcknowledgementTest.java
===================================================================
--- trunk/tests/jms-tests/src/org/jboss/test/messaging/jms/AcknowledgementTest.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/tests/jms-tests/src/org/jboss/test/messaging/jms/AcknowledgementTest.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -128,28 +128,28 @@
 
       try
       {
-         // conn = cf.createTopicConnection();
-         // TopicSession sess = conn.createTopicSession(true, 0);
-         // TopicPublisher pub = sess.createPublisher(topic1);
-         // TopicSubscriber cons = sess.createSubscriber(topic1);
-         // conn.start();
-         //
-         // Message m = sess.createTextMessage("testing123");
-         // pub.publish(m);
-         // sess.commit();
-         //
-         // TextMessage m2 = (TextMessage) cons.receive(3000);
-         // assertNotNull(m2);
-         // assertEquals("testing123", m2.getText());
-         //
-         // sess.rollback();
-         //
-         // m2 = (TextMessage) cons.receive(3000);
-         // assertNotNull(m2);
-         // assertEquals("testing123", m2.getText());
-         //
-         // conn.close();
+         conn = cf.createTopicConnection();
+         TopicSession sess = conn.createTopicSession(true, 0);
+         TopicPublisher pub = sess.createPublisher(topic1);
+         TopicSubscriber cons = sess.createSubscriber(topic1);
+         conn.start();
 
+         Message m = sess.createTextMessage("testing123");
+         pub.publish(m);
+         sess.commit();
+
+         TextMessage m2 = (TextMessage)cons.receive(3000);
+         assertNotNull(m2);
+         assertEquals("testing123", m2.getText());
+
+         sess.rollback();
+
+         m2 = (TextMessage)cons.receive(3000);
+         assertNotNull(m2);
+         assertEquals("testing123", m2.getText());
+
+         conn.close();
+
          conn = cf.createTopicConnection();
          conn.start();
 
@@ -1046,7 +1046,7 @@
          log.trace("Set message listener");
 
          listener.waitForMessages();
-         
+
          log.info("Waited for messages");
 
          // Recover forces an ack so there will be one
@@ -1206,7 +1206,7 @@
             count++;
 
             TextMessage tm = (TextMessage)m;
-            
+
             log.info("got message " + tm.getText());
 
             // Receive first three messages then recover() session
@@ -1280,7 +1280,7 @@
             count++;
 
             TextMessage tm = (TextMessage)m;
-            
+
             log.info("Got message " + tm.getText());
 
             // Receive first three messages then recover() session
@@ -1341,7 +1341,6 @@
 
    private class MessageListenerClientAck extends LatchListener
    {
-
       MessageListenerClientAck(final Session sess)
       {
          super(sess);
@@ -1355,7 +1354,7 @@
             count++;
 
             TextMessage tm = (TextMessage)m;
-            
+
             log.info("Got message " + tm.getText());
 
             if (count == 1)

Added: trunk/tests/src/org/jboss/messaging/tests/integration/cluster/MultiThreadRandomFailoverTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/cluster/MultiThreadRandomFailoverTest.java	                        (rev 0)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/cluster/MultiThreadRandomFailoverTest.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -0,0 +1,1485 @@
+/*
+ * JBoss, Home of Professional Open Source Copyright 2005-2008, Red Hat Middleware LLC, and individual contributors by
+ * the @authors tag. See the copyright.txt in the distribution for a full listing of individual contributors. This is
+ * free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version.
+ * This software is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details. You should have received a copy of the GNU Lesser General Public License along with this software; if not,
+ * write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF
+ * site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.tests.integration.cluster;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import junit.framework.TestCase;
+
+import org.jboss.messaging.core.client.ClientConsumer;
+import org.jboss.messaging.core.client.ClientMessage;
+import org.jboss.messaging.core.client.ClientProducer;
+import org.jboss.messaging.core.client.ClientSession;
+import org.jboss.messaging.core.client.ClientSessionFactory;
+import org.jboss.messaging.core.client.MessageHandler;
+import org.jboss.messaging.core.client.impl.ClientSessionFactoryImpl;
+import org.jboss.messaging.core.client.impl.ClientSessionImpl;
+import org.jboss.messaging.core.config.Configuration;
+import org.jboss.messaging.core.config.TransportConfiguration;
+import org.jboss.messaging.core.config.impl.ConfigurationImpl;
+import org.jboss.messaging.core.exception.MessagingException;
+import org.jboss.messaging.core.logging.Logger;
+import org.jboss.messaging.core.remoting.impl.ConnectionRegistryImpl;
+import org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl;
+import org.jboss.messaging.core.remoting.impl.invm.InVMRegistry;
+import org.jboss.messaging.core.remoting.impl.invm.TransportConstants;
+import org.jboss.messaging.core.server.MessagingService;
+import org.jboss.messaging.core.server.impl.MessagingServiceImpl;
+import org.jboss.messaging.jms.client.JBossTextMessage;
+import org.jboss.messaging.util.SimpleString;
+
+/**
+ * A MultiThreadRandomFailoverTest
+ * 
+ * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
+ */
+public class MultiThreadRandomFailoverTest extends TestCase
+{
+   private static final Logger log = Logger.getLogger(MultiThreadRandomFailoverTest.class);
+
+   // Constants -----------------------------------------------------
+
+   private static final int RECEIVE_TIMEOUT = 5000;
+   
+   private static final int NUM_THREADS = 10;
+
+   // Attributes ----------------------------------------------------
+   private static final SimpleString ADDRESS = new SimpleString("FailoverTestAddress");
+
+   private MessagingService liveService;
+
+   private MessagingService backupService;
+
+   private final Map<String, Object> backupParams = new HashMap<String, Object>();
+
+   private Timer timer = new Timer();
+
+   // Static --------------------------------------------------------
+
+   // Constructors --------------------------------------------------
+
+   // Public --------------------------------------------------------
+
+
+   public void testA() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestA(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+   
+   public void testB() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestB(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+   
+   public void testC() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestC(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+   
+   public void testD() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestD(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+
+   public void testE() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestE(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+
+   public void testF() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestF(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+
+   public void testG() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestG(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+   
+   public void testH() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestH(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+   
+   public void testI() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestI(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+   
+   public void testJ() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestJ(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+   
+   public void testK() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestK(sf, threadNum);
+         }
+      }, NUM_THREADS);
+   }
+   
+   public void testL() throws Exception
+   {
+      runTestMultipleThreads(new RunnableTest()
+      {
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestL(sf);
+         }
+      }, NUM_THREADS);
+   }
+   
+   // Package protected ---------------------------------------------
+
+   // Protected -----------------------------------------------------
+   
+   protected void doTestA(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, true, true, false);
+
+         sessConsume.start();
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, true, true, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      class MyHandler implements MessageHandler
+      {
+         final CountDownLatch latch = new CountDownLatch(1);
+
+         volatile int count;
+
+         public void onMessage(ClientMessage message)
+         {
+            try
+            {
+               message.processed();
+            }
+            catch (MessagingException me)
+            {
+               log.error("Failed to process", me);
+            }
+            
+            if (count >= numMessages)
+            {
+               return;
+            }
+
+            count++;
+            
+            if (count == numMessages)
+            {
+               latch.countDown();
+            }
+         }
+      }
+
+      Set<MyHandler> handlers = new HashSet<MyHandler>();
+
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler();
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(5000, TimeUnit.MILLISECONDS);
+
+         assertTrue("Didn't receive all messages", ok);
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestB(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, true, true, false);
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, true, true, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.start();
+      }
+
+      class MyHandler implements MessageHandler
+      {
+         final CountDownLatch latch = new CountDownLatch(1);
+
+         volatile int count;
+
+         public void onMessage(ClientMessage message)
+         {
+            try
+            {
+               message.processed();
+            }
+            catch (MessagingException me)
+            {
+               log.error("Failed to process", me);
+            }
+            
+            if (count >= numMessages)
+            {
+               return;
+            }
+
+            count++;
+            
+            if (count == numMessages)
+            {
+               latch.countDown();
+            }
+         }
+      }
+
+      Set<MyHandler> handlers = new HashSet<MyHandler>();
+
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler();
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
+
+         assertTrue(ok);
+      }
+
+      sessSend.close();
+
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+
+   }
+
+   protected void doTestC(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, false, false, false);
+
+         sessConsume.start();
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, true, true, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      sessSend.rollback();
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      sessSend.commit();
+
+      class MyHandler implements MessageHandler
+      {
+         final CountDownLatch latch = new CountDownLatch(1);
+
+         volatile int count;
+
+         public void onMessage(ClientMessage message)
+         {
+            try
+            {
+               message.processed();
+            }
+            catch (MessagingException me)
+            {
+               log.error("Failed to process", me);
+            }
+            
+            if (count >= numMessages)
+            {
+               return;
+            }
+
+            count++;
+            
+            if (count == numMessages)
+            {
+               latch.countDown();
+            }
+         }
+      }
+
+      Set<MyHandler> handlers = new HashSet<MyHandler>();
+
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler();
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
+
+         assertTrue(ok);
+      }
+
+      handlers.clear();
+
+      // New handlers
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler();
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.rollback();
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
+
+         assertTrue(ok);
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.commit();
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestD(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + " sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, false, false, false);
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, true, true, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      sessSend.rollback();
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      sessSend.commit();
+
+      for (ClientSession session : sessions)
+      {
+         session.start();
+      }
+
+      class MyHandler implements MessageHandler
+      {
+         final CountDownLatch latch = new CountDownLatch(1);
+
+         volatile int count;
+
+         public void onMessage(ClientMessage message)
+         {
+            try
+            {
+               message.processed();
+            }
+            catch (MessagingException me)
+            {
+               log.error("Failed to process", me);
+            }
+            
+            if (count >= numMessages)
+            {
+               return;
+            }
+
+            count++;
+            
+            if (count == numMessages)
+            {
+               latch.countDown();
+            }
+         }
+      }
+
+      Set<MyHandler> handlers = new HashSet<MyHandler>();
+
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler();
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
+
+         assertTrue(ok);
+      }
+
+      handlers.clear();
+
+      // New handlers
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler();
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.rollback();
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
+
+         assertTrue(ok);
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.commit();
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + " sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   // Now with synchronous receive()
+
+   protected void doTestE(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, true, true, false);
+
+         sessConsume.start();
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, true, true, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         for (ClientConsumer consumer : consumers)
+         {
+            ClientMessage msg = consumer.receive(RECEIVE_TIMEOUT);
+
+            assertNotNull(msg);
+
+            msg.processed();
+         }
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestF(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, true, true, false);
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, true, true, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.start();
+      }
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         for (ClientConsumer consumer : consumers)
+         {
+            ClientMessage msg = consumer.receive(RECEIVE_TIMEOUT);
+
+            assertNotNull(msg);
+
+            msg.processed();
+         }
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestG(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, false, false, false);
+
+         sessConsume.start();
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, false, false, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      sessSend.rollback();
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      sessSend.commit();
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         for (ClientConsumer consumer : consumers)
+         {
+            ClientMessage msg = consumer.receive(RECEIVE_TIMEOUT);
+
+            assertNotNull(msg);
+
+            msg.processed();
+         }
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.rollback();
+      }
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         for (ClientConsumer consumer : consumers)
+         {
+            ClientMessage msg = consumer.receive(RECEIVE_TIMEOUT);
+
+            assertNotNull(msg);
+
+            msg.processed();
+         }
+      }
+
+
+      for (ClientSession session : sessions)
+      {
+         session.commit();
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestH(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, false, false, false);
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, false, false, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      sessSend.rollback();
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("count"), i);
+         message.getBody().flip();
+         producer.send(message);
+      }
+
+      sessSend.commit();
+
+      for (ClientSession session : sessions)
+      {
+         session.start();
+      }
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         for (ClientConsumer consumer : consumers)
+         {            
+            ClientMessage msg = consumer.receive(RECEIVE_TIMEOUT);
+
+            msg.processed();
+         }
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.rollback();
+      }
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         for (ClientConsumer consumer : consumers)
+         {
+            ClientMessage msg = consumer.receive(RECEIVE_TIMEOUT);
+
+            msg.processed();
+         }
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.commit();
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestI(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      ClientSession sessCreate = sf.createSession(false, true, true, false);
+
+      sessCreate.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
+
+      ClientSession sess = sf.createSession(false, true, true, false);
+
+      sess.start();
+
+      ClientConsumer consumer = sess.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
+
+      ClientProducer producer = sess.createProducer(ADDRESS);
+
+      ClientMessage message = sess.createClientMessage(JBossTextMessage.TYPE,
+                                                       false,
+                                                       0,
+                                                       System.currentTimeMillis(),
+                                                       (byte)1);
+      message.getBody().flip();
+
+      producer.send(message);
+
+      ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
+
+      assertNotNull(message2);
+
+      message2.processed();
+
+      sess.close();
+
+      sessCreate.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
+
+      sessCreate.close();
+   }
+
+   protected void doTestJ(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      ClientSession sessCreate = sf.createSession(false, true, true, false);
+
+      sessCreate.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
+
+      ClientSession sess = sf.createSession(false, true, true, false);
+
+      sess.start();
+
+      ClientConsumer consumer = sess.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
+
+      ClientProducer producer = sess.createProducer(ADDRESS);
+
+      ClientMessage message = sess.createClientMessage(JBossTextMessage.TYPE,
+                                                       false,
+                                                       0,
+                                                       System.currentTimeMillis(),
+                                                       (byte)1);
+      message.getBody().flip();
+
+      producer.send(message);
+
+      ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
+
+      assertNotNull(message2);
+
+      message2.processed();
+
+      sess.close();
+
+      sessCreate.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
+
+      sessCreate.close();
+   }
+
+   protected void doTestK(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      ClientSession s = sf.createSession(false, false, false, false);
+
+      s.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
+
+      final int numConsumers = 100;
+
+      for (int i = 0; i < numConsumers; i++)
+      {
+         ClientConsumer consumer = s.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
+
+         consumer.close();
+      }
+
+      s.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
+
+      s.close();
+   }
+
+   protected void doTestL(final ClientSessionFactory sf) throws Exception
+   {
+      ClientSession s = sf.createSession(false, false, false, false);
+
+      final int numSessions = 100;
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         ClientSession session = sf.createSession(false, false, false, false);
+
+         session.close();
+      }
+
+      s.close();
+   }
+
+   protected int getNumIterations()
+   {
+      return 20;
+   }
+   
+   protected void setUp() throws Exception
+   {
+      super.setUp();
+      
+      log.info("************ Starting test " + this.getName());
+   }
+
+   // Private -------------------------------------------------------
+
+   private void runTestMultipleThreads(final RunnableTest runnable, final int numThreads) throws Exception
+   {
+      final int numIts = getNumIterations();
+
+      for (int its = 0; its < numIts; its++)
+      {
+         log.info("************ ITERATION: " + its);
+         start();
+
+         final ClientSessionFactoryImpl sf = new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"),
+                                                                          new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
+                                                                                                     backupParams));
+
+         ClientSession session = sf.createSession(false, false, false, false);
+
+         Failer failer = startFailer(1000, session);
+
+         class Runner extends Thread
+         {
+            private volatile Throwable throwable;
+
+            private final RunnableTest test;
+
+            private final int threadNum;
+
+            Runner(final RunnableTest test, final int threadNum)
+            {
+               this.test = test;
+
+               this.threadNum = threadNum;
+            }
+
+            public void run()
+            {
+               try
+               {
+                  test.run(sf, threadNum);
+               }
+               catch (Throwable t)
+               {
+                  throwable = t;
+
+                  log.error("Failed to run test", t);
+               }
+            }
+         }
+         
+         do
+         {
+            List<Runner> threads = new ArrayList<Runner>();
+            
+            for (int i = 0; i < numThreads; i++)
+            {
+               Runner runner = new Runner(runnable, i);
+
+               threads.add(runner);
+
+               runner.start();
+            }
+            
+            for (Runner thread : threads)
+            {
+               thread.join();               
+
+               assertNull(thread.throwable);
+            }
+            
+            runnable.checkFail();
+         }
+         while (!failer.isExecuted());
+         
+         session.close();
+
+         assertEquals(0, sf.getSessionCount());
+
+         stop();
+      }
+   }
+   
+   private Failer startFailer(final long time, final ClientSession session)
+   {
+      Failer failer = new Failer(session);
+
+      timer.schedule(failer, (long)(time * Math.random()), 100);
+      
+      return failer;
+   }
+
+   private void start() throws Exception
+   {      
+      Configuration backupConf = new ConfigurationImpl();
+      backupConf.setSecurityEnabled(false);
+      backupConf.setPacketConfirmationBatchSize(10);
+      backupParams.put(TransportConstants.SERVER_ID_PROP_NAME, 1);
+      backupConf.getAcceptorConfigurations()
+                .add(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMAcceptorFactory",
+                                                backupParams));
+      backupConf.setBackup(true);
+      backupService = MessagingServiceImpl.newNullStorageMessagingServer(backupConf);
+      backupService.start();
+
+      // We need to sleep > 16 ms otherwise the id generators on live and backup could be initialised
+      // with the same time component
+      Thread.sleep(17);
+
+      Configuration liveConf = new ConfigurationImpl();
+      liveConf.setSecurityEnabled(false);
+      liveConf.setPacketConfirmationBatchSize(10);
+      liveConf.getAcceptorConfigurations()
+              .add(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMAcceptorFactory"));
+      liveConf.setBackupConnectorConfiguration(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
+                                                                          backupParams));
+      liveService = MessagingServiceImpl.newNullStorageMessagingServer(liveConf);
+      liveService.start();
+   }
+
+   private void stop() throws Exception
+   {
+      ConnectionRegistryImpl.instance.dump();
+
+      assertEquals(0, ConnectionRegistryImpl.instance.size());
+
+      // ConnectionRegistryImpl.instance.clear();
+
+      assertEquals(0, backupService.getServer().getRemotingService().getConnections().size());
+
+      backupService.stop();
+
+      assertEquals(0, liveService.getServer().getRemotingService().getConnections().size());
+
+      liveService.stop();
+
+      assertEquals(0, InVMRegistry.instance.size());
+   }
+
+   // Inner classes -------------------------------------------------
+
+   class Failer extends TimerTask
+   {
+      private final ClientSession session;
+
+      private boolean executed;
+
+      public Failer(final ClientSession session)
+      {
+         this.session = session;
+      }
+
+      public synchronized void run()
+      {
+         log.info("** Failing connection");
+
+         RemotingConnectionImpl conn = (RemotingConnectionImpl)((ClientSessionImpl)session).getConnection();
+
+         conn.fail(new MessagingException(MessagingException.NOT_CONNECTED, "blah"));
+
+         log.info("** Fail complete");
+
+         cancel();
+
+         executed = true;
+      }
+
+      public synchronized boolean isExecuted()
+      {
+         return executed;
+      }
+   }
+
+   public abstract class RunnableTest extends Thread
+   {
+      private volatile String failReason;
+      private volatile Throwable throwable;
+      
+      public void setFailed(final String reason, final Throwable throwable)
+      {
+         this.failReason = reason;
+         this.throwable = throwable;
+      }
+      
+      public void checkFail()
+      {
+         if (throwable != null)
+         {
+            log.error("Test failed: " + failReason, throwable);
+         }
+         if (failReason != null)
+         {
+            fail(failReason);
+         }
+      }
+      public abstract void run(final ClientSessionFactory sf, final int threadNum) throws Exception;
+   }
+}

Modified: trunk/tests/src/org/jboss/messaging/tests/integration/cluster/RandomFailoverTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/cluster/RandomFailoverTest.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/cluster/RandomFailoverTest.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -46,7 +46,7 @@
 import org.jboss.messaging.util.SimpleString;
 
 /**
- * A RandomFailoverTest
+ * A RandomFailoverSoakTest
  * 
  * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
  */
@@ -67,6 +67,8 @@
    private MessagingService backupService;
 
    private final Map<String, Object> backupParams = new HashMap<String, Object>();
+   
+   private Timer timer = new Timer();
 
    // Static --------------------------------------------------------
 
@@ -74,61 +76,14 @@
 
    // Public --------------------------------------------------------
 
-   private Timer timer = new Timer();
-
-   private volatile Failer failer;
-
-   private void startFailer(final long time, final ClientSession session)
-   {
-      failer = new Failer(session);
-
-      timer.schedule(failer, (long)(time * Math.random()), 100);
-   }
-
-   private class Failer extends TimerTask
-   {
-      private final ClientSession session;
-
-      private boolean executed;
-
-      public Failer(final ClientSession session)
-      {
-         this.session = session;
-      }
-
-      public synchronized void run()
-      {
-         log.info("** Failing connection");
-
-         RemotingConnectionImpl conn = (RemotingConnectionImpl)((ClientSessionImpl)session).getConnection();
-
-         conn.fail(new MessagingException(MessagingException.NOT_CONNECTED, "blah"));
-
-         log.info("** Fail complete");
-
-         cancel();
-
-         executed = true;
-      }
-
-      public synchronized boolean isExecuted()
-      {
-         return executed;
-      }
-   }
    
-   private interface RunnableTest
-   {
-      void run(final ClientSessionFactory sf) throws Exception;      
-   }
-   
    public void testA() throws Exception
    {
       runTest(new RunnableTest()
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testA(sf);
+            doTestA(sf);
          }
       });
    }
@@ -139,7 +94,7 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testB(sf);
+            doTestB(sf);
          }
       });
    }
@@ -150,7 +105,7 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testC(sf);
+            doTestC(sf);
          }
       });
    }
@@ -161,7 +116,7 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testD(sf);
+            doTestD(sf);
          }
       });
    }
@@ -172,7 +127,7 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testE(sf);
+            doTestE(sf);
          }
       });
    }
@@ -183,7 +138,7 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testF(sf);
+            doTestF(sf);
          }
       });
    }
@@ -194,7 +149,7 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testG(sf);
+            doTestG(sf);
          }
       });
    }
@@ -205,7 +160,7 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testH(sf);
+            doTestH(sf);
          }
       });
    }
@@ -216,7 +171,7 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testI(sf);
+            doTestI(sf);
          }
       });
    }
@@ -227,7 +182,7 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testJ(sf);
+            doTestJ(sf);
          }
       });
    }
@@ -238,7 +193,7 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testK(sf);
+            doTestK(sf);
          }
       });
    }
@@ -249,14 +204,14 @@
       {
          public void run(final ClientSessionFactory sf) throws Exception
          {
-            testK(sf);
+            doTestK(sf);
          }
       });
    }
 
    public void runTest(final RunnableTest runnable) throws Exception
    {
-      final int numIts = 100;
+      final int numIts = getNumIterations();
       
       for (int its = 0; its < numIts; its++)
       {
@@ -268,7 +223,7 @@
 
          ClientSession session = sf.createSession(false, false, false, false);
 
-         startFailer(1000, session);
+         Failer failer = startFailer(1000, session);
 
          do
          {
@@ -283,8 +238,12 @@
          stop();
       }
    }
+       
+   // Package protected ---------------------------------------------
 
-   public void testA(final ClientSessionFactory sf) throws Exception
+   // Protected -----------------------------------------------------
+   
+   protected void doTestA(final ClientSessionFactory sf) throws Exception
    {
       long start = System.currentTimeMillis();
 
@@ -401,7 +360,7 @@
       log.info("duration " + (end - start));
    }
 
-   public void testB(final ClientSessionFactory sf) throws Exception
+   protected void doTestB(final ClientSessionFactory sf) throws Exception
    {
       long start = System.currentTimeMillis();
 
@@ -514,7 +473,7 @@
 
    }
 
-   public void testC(final ClientSessionFactory sf) throws Exception
+   protected void doTestC(final ClientSessionFactory sf) throws Exception
    {
       long start = System.currentTimeMillis();
 
@@ -667,7 +626,7 @@
       log.info("duration " + (end - start));
    }
 
-   public void testD(final ClientSessionFactory sf) throws Exception
+   protected void doTestD(final ClientSessionFactory sf) throws Exception
    {
       long start = System.currentTimeMillis();
 
@@ -825,7 +784,7 @@
 
    // Now with synchronous receive()
 
-   public void testE(final ClientSessionFactory sf) throws Exception
+   protected void doTestE(final ClientSessionFactory sf) throws Exception
    {
       long start = System.currentTimeMillis();
 
@@ -915,7 +874,7 @@
       log.info("duration " + (end - start));
    }
 
-   public void testF(final ClientSessionFactory sf) throws Exception
+   protected void doTestF(final ClientSessionFactory sf) throws Exception
    {
       long start = System.currentTimeMillis();
 
@@ -1008,7 +967,7 @@
       log.info("duration " + (end - start));
    }
 
-   public void testG(final ClientSessionFactory sf) throws Exception
+   protected void doTestG(final ClientSessionFactory sf) throws Exception
    {
       long start = System.currentTimeMillis();
 
@@ -1145,7 +1104,7 @@
       log.info("duration " + (end - start));
    }
 
-   public void testH(final ClientSessionFactory sf) throws Exception
+   protected void doTestH(final ClientSessionFactory sf) throws Exception
    {
       long start = System.currentTimeMillis();
 
@@ -1288,7 +1247,7 @@
       log.info("duration " + (end - start));
    }
 
-   public void testI(final ClientSessionFactory sf) throws Exception
+   protected void doTestI(final ClientSessionFactory sf) throws Exception
    {
       ClientSession sessCreate = sf.createSession(false, true, true, false);
 
@@ -1324,7 +1283,7 @@
       sessCreate.close();     
    }
 
-   public void testJ(final ClientSessionFactory sf) throws Exception
+   protected void doTestJ(final ClientSessionFactory sf) throws Exception
    {
       ClientSession sessCreate = sf.createSession(false, true, true, false);
 
@@ -1360,7 +1319,7 @@
       sessCreate.close();
    }
 
-   public void testK(final ClientSessionFactory sf) throws Exception
+   protected void doTestK(final ClientSessionFactory sf) throws Exception
    {
       ClientSession s = sf.createSession(false, false, false, false);
 
@@ -1380,7 +1339,7 @@
       s.close();
    }
 
-   public void testL(final ClientSessionFactory sf) throws Exception
+   protected void doTestL(final ClientSessionFactory sf) throws Exception
    {
       ClientSession s = sf.createSession(false, false, false, false);
 
@@ -1395,11 +1354,24 @@
 
       s.close();
    }
+   
+   protected int getNumIterations()
+   {
+      return 10;
+   }
+   
+  
+   // Private -------------------------------------------------------
+   
+   private Failer startFailer(final long time, final ClientSession session)
+   {
+      Failer failer = new Failer(session);
 
-   // Package protected ---------------------------------------------
-
-   // Protected -----------------------------------------------------
-
+      timer.schedule(failer, (long)(time * Math.random()), 100);
+      
+      return failer;
+   }
+   
    private void start() throws Exception
    {
       Configuration backupConf = new ConfigurationImpl();
@@ -1446,8 +1418,43 @@
 
       assertEquals(0, InVMRegistry.instance.size());
    }
+  
+   // Inner classes -------------------------------------------------
+   
+   class Failer extends TimerTask
+   {
+      private final ClientSession session;
 
-   // Private -------------------------------------------------------
+      private boolean executed;
 
-   // Inner classes -------------------------------------------------
+      public Failer(final ClientSession session)
+      {
+         this.session = session;
+      }
+
+      public synchronized void run()
+      {
+         log.info("** Failing connection");
+
+         RemotingConnectionImpl conn = (RemotingConnectionImpl)((ClientSessionImpl)session).getConnection();
+
+         conn.fail(new MessagingException(MessagingException.NOT_CONNECTED, "blah"));
+
+         log.info("** Fail complete");
+
+         cancel();
+
+         executed = true;
+      }
+
+      public synchronized boolean isExecuted()
+      {
+         return executed;
+      }
+   }
+   
+   public abstract class RunnableTest
+   {
+      abstract void run(final ClientSessionFactory sf) throws Exception;      
+   }
 }

Modified: trunk/tests/src/org/jboss/messaging/tests/integration/cluster/SimpleManualFailoverTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/cluster/SimpleManualFailoverTest.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/cluster/SimpleManualFailoverTest.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -137,6 +137,7 @@
       {
          public void connectionFailed(MessagingException me)
          {
+            log.info("*** connection failed");
             latch.countDown();
          }
       }

Deleted: trunk/tests/src/org/jboss/messaging/tests/integration/cluster/SystematicFailoverTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/cluster/SystematicFailoverTest.java	2008-10-16 17:23:43 UTC (rev 5122)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/cluster/SystematicFailoverTest.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -1,574 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source Copyright 2005-2008, Red Hat Middleware LLC, and individual contributors by
- * the @authors tag. See the copyright.txt in the distribution for a full listing of individual contributors. This is
- * free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as
- * published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version.
- * This software is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
- * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
- * details. You should have received a copy of the GNU Lesser General Public License along with this software; if not,
- * write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF
- * site: http://www.fsf.org.
- */
-
-package org.jboss.messaging.tests.integration.cluster;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import junit.framework.TestCase;
-
-import org.jboss.messaging.core.client.ClientConsumer;
-import org.jboss.messaging.core.client.ClientMessage;
-import org.jboss.messaging.core.client.ClientProducer;
-import org.jboss.messaging.core.client.ClientSession;
-import org.jboss.messaging.core.client.ClientSessionFactory;
-import org.jboss.messaging.core.client.MessageHandler;
-import org.jboss.messaging.core.client.impl.ClientSessionFactoryImpl;
-import org.jboss.messaging.core.client.impl.ClientSessionImpl;
-import org.jboss.messaging.core.config.Configuration;
-import org.jboss.messaging.core.config.TransportConfiguration;
-import org.jboss.messaging.core.config.impl.ConfigurationImpl;
-import org.jboss.messaging.core.exception.MessagingException;
-import org.jboss.messaging.core.logging.Logger;
-import org.jboss.messaging.core.remoting.RemotingConnection;
-import org.jboss.messaging.core.remoting.impl.ConnectionRegistryImpl;
-import org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl;
-import org.jboss.messaging.core.remoting.impl.invm.InVMRegistry;
-import org.jboss.messaging.core.remoting.impl.invm.TransportConstants;
-import org.jboss.messaging.core.server.MessagingService;
-import org.jboss.messaging.core.server.impl.MessagingServiceImpl;
-import org.jboss.messaging.jms.client.JBossTextMessage;
-import org.jboss.messaging.util.SimpleString;
-
-/**
- * A SystematicFailoverTest
- * 
- * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
- */
-public class SystematicFailoverTest extends TestCase
-{
-   private static final Logger log = Logger.getLogger(SimpleAutomaticFailoverTest.class);
-
-   // Constants -----------------------------------------------------
-
-   // Attributes ----------------------------------------------------
-
-   private static final SimpleString ADDRESS = new SimpleString("FailoverTestAddress");
-
-   private MessagingService liveService;
-
-   private MessagingService backupService;
-
-   private final Map<String, Object> backupParams = new HashMap<String, Object>();
-
-   // Static --------------------------------------------------------
-
-   // Constructors --------------------------------------------------
-
-   // Public --------------------------------------------------------
-
-   private volatile Thread failThread;
-
-   private void setFailAfterRandomTime(final RemotingConnection conn)
-   {
-      failThread = new Thread()
-      {
-         public void run()
-         {
-            try
-            {
-               Thread.sleep((long)(300 * Math.random()));
-
-               log.info("Failing");
-               conn.fail(new MessagingException(MessagingException.NOT_CONNECTED, "blah"));
-            }
-            catch (InterruptedException ignore)
-            {
-            }
-         }
-      };
-
-      failThread.start();
-   }
-
-   private void waitForFailThread() throws Exception
-   {
-      if (failThread != null)
-      {
-         failThread.join();
-      }
-   }
-
-   public void testExerciseAPI() throws Exception
-   {
-      for (int its = 0; its < 1000; its++)
-      {         
-         start();
-
-         long start = System.currentTimeMillis();
-
-         ClientSessionFactory sf = new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"),
-                                                                new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
-                                                                                           backupParams));
-
-         // Non transacted
-         ClientSession session1 = sf.createSession(false, true, true, false);
-
-         RemotingConnectionImpl conn = (RemotingConnectionImpl)((ClientSessionImpl)session1).getConnection();
-
-         setFailAfterRandomTime(conn);
-
-         session1.createQueue(ADDRESS, ADDRESS, null, false, false);
-
-         final int numMessages = 1000;
-
-         ClientProducer producer1 = session1.createProducer(ADDRESS);
-
-         for (int i = 0; i < numMessages; i++)
-         {
-            ClientMessage message = session1.createClientMessage(JBossTextMessage.TYPE,
-                                                                 false,
-                                                                 0,
-                                                                 System.currentTimeMillis(),
-                                                                 (byte)1);
-            message.putIntProperty(new SimpleString("count"), i);
-            message.getBody().putString("aardvarks");
-            message.getBody().flip();
-            producer1.send(message);
-         }
-
-         ClientSession session2 = sf.createSession(false, true, true, false);
-
-         session2.start();
-
-         ClientConsumer consumer1 = session2.createConsumer(ADDRESS);
-
-         for (int i = 0; i < numMessages; i++)
-         {
-            ClientMessage message = consumer1.receive();
-
-            assertEquals("aardvarks", message.getBody().getString());
-            assertEquals(i, message.getProperty(new SimpleString("count")));
-
-            message.processed();
-         }
-
-         producer1.close();
-
-         ClientProducer producer2 = session1.createProducer(ADDRESS);
-
-         for (int i = 0; i < numMessages; i++)
-         {
-            ClientMessage message = session2.createClientMessage(JBossTextMessage.TYPE,
-                                                                 false,
-                                                                 0,
-                                                                 System.currentTimeMillis(),
-                                                                 (byte)1);
-            message.putIntProperty(new SimpleString("count"), i);
-            message.getBody().putString("aardvarks");
-            message.getBody().flip();
-            producer2.send(message);
-         }
-
-         for (int i = 0; i < numMessages; i++)
-         {
-            ClientMessage message = consumer1.receive();
-
-            assertEquals("aardvarks", message.getBody().getString());
-            assertEquals(i, message.getProperty(new SimpleString("count")));
-
-            message.processed();
-         }
-
-         consumer1.close();
-
-         // Transacted
-         ClientSession session3 = sf.createSession(false, false, false, false);
-
-         ClientProducer producer3 = session3.createProducer(ADDRESS);
-
-         for (int i = 0; i < numMessages; i++)
-         {
-            ClientMessage message = session3.createClientMessage(JBossTextMessage.TYPE,
-                                                                 false,
-                                                                 0,
-                                                                 System.currentTimeMillis(),
-                                                                 (byte)1);
-            message.putIntProperty(new SimpleString("count"), i);
-            message.getBody().putString("aardvarks");
-            message.getBody().flip();
-            producer3.send(message);
-
-            session3.rollback();
-
-            message = session3.createClientMessage(JBossTextMessage.TYPE, false, 0, System.currentTimeMillis(), (byte)1);
-            message.putIntProperty(new SimpleString("count"), i);
-            message.getBody().putString("aardvarks");
-            message.getBody().flip();
-            producer3.send(message);
-
-            session3.commit();
-         }
-
-         ClientConsumer consumer2 = session3.createConsumer(ADDRESS);
-
-         session3.start();
-
-         for (int i = 0; i < numMessages; i++)
-         {
-            ClientMessage message = consumer2.receive();
-
-            assertEquals("aardvarks", message.getBody().getString());
-            assertEquals(i, message.getProperty(new SimpleString("count")));
-
-            message.processed();
-         }
-
-         session3.rollback();
-
-         for (int i = 0; i < numMessages; i++)
-         {
-            ClientMessage message = consumer2.receive();
-
-            assertEquals("aardvarks", message.getBody().getString());
-            assertEquals(i, message.getProperty(new SimpleString("count")));
-
-            message.processed();
-         }
-
-         session3.commit();
-
-         session1.close();
-
-         session2.close();
-
-         final int numConsumers = 10;
-
-         Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
-
-         final ClientSession session4 = sf.createSession(false, true, true, false);
-
-         session4.start();
-
-         for (int i = 0; i < numConsumers; i++)
-         {
-            SimpleString wibble = new SimpleString("wibble");
-
-            session4.addDestination(wibble, false, false);
-
-            session4.removeDestination(wibble, false);
-
-            SimpleString subName = new SimpleString("sub" + i);
-
-            session4.createQueue(ADDRESS, subName, null, false, false);
-
-            ClientConsumer consumer = session4.createConsumer(subName);
-
-            consumers.add(consumer);
-         }
-
-         ClientProducer producer4 = session4.createProducer(ADDRESS);
-
-         for (int i = 0; i < numMessages; i++)
-         {
-            ClientMessage message = session4.createClientMessage(JBossTextMessage.TYPE,
-                                                                 false,
-                                                                 0,
-                                                                 System.currentTimeMillis(),
-                                                                 (byte)1);
-            message.putIntProperty(new SimpleString("count"), i);
-            message.getBody().putString("aardvarks");
-            message.getBody().flip();
-            producer4.send(message);
-         }
-
-         // Consume synchronously
-         for (ClientConsumer consumer : consumers)
-         {
-            for (int i = 0; i < numMessages; i++)
-            {
-               ClientMessage message = consumer.receive();
-
-               assertEquals("aardvarks", message.getBody().getString());
-               assertEquals(i, message.getProperty(new SimpleString("count")));
-
-               message.processed();
-            }
-         }
-
-         class MyHandler implements MessageHandler
-         {
-            final CountDownLatch latch = new CountDownLatch(1);
-
-            volatile int count;
-
-            public void onMessage(ClientMessage message)
-            {
-               assertEquals(count, message.getProperty(new SimpleString("count")));
-
-               // log.info(this + " got message " + count);
-
-               try
-               {
-                  message.processed();
-               }
-               catch (MessagingException me)
-               {
-                  fail();
-               }
-
-               count++;
-
-               if (count == numMessages)
-               {
-                  latch.countDown();
-               }
-            }
-         }
-
-         Set<MyHandler> handlers = new HashSet<MyHandler>();
-
-         for (ClientConsumer consumer : consumers)
-         {
-            MyHandler handler = new MyHandler();
-
-            consumer.setMessageHandler(handler);
-
-            handlers.add(handler);
-         }
-
-         for (int i = 0; i < numMessages; i++)
-         {
-            ClientMessage message = session4.createClientMessage(JBossTextMessage.TYPE,
-                                                                 false,
-                                                                 0,
-                                                                 System.currentTimeMillis(),
-                                                                 (byte)1);
-            message.putIntProperty(new SimpleString("count"), i);
-            message.getBody().putString("aardvarks");
-            message.getBody().flip();
-            producer4.send(message);
-         }
-
-         for (MyHandler handler : handlers)
-         {
-            boolean ok = handler.latch.await(1000, TimeUnit.MILLISECONDS);
-
-            assertTrue(ok);
-         }
-
-         session3.close();
-
-         session4.close();
-
-         long end = System.currentTimeMillis();
-
-         log.info("iteration" + its + " duration " + (end - start));
-
-         this.waitForFailThread();
-
-         stop();
-      }
-   }
-
-   public void testReplicateWithHandlers() throws Exception
-   {
-      for (int its = 0; its < 100; its++)
-      {
-         log.info("Starting iteration " + its);
-
-         start();
-
-         long start = System.currentTimeMillis();
-
-         ClientSessionFactory sf = new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"),
-                                                                new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
-                                                                                           backupParams));
-
-         final int numMessages = 100;
-
-         final int numConsumers = 10;
-
-         Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
-
-         ClientSession sessConsume = sf.createSession(false, true, true, false);
-         
-         sessConsume.start();
-         
-         for (int i = 0; i < numConsumers; i++)
-         {
-            SimpleString subName = new SimpleString("sub" + i);
-
-            sessConsume.createQueue(ADDRESS, subName, null, false, false);
-
-            ClientConsumer consumer = sessConsume.createConsumer(subName);
-
-            consumers.add(consumer);
-         }
-         
-         ClientSession sessSend = sf.createSession(false, true, true, false);         
-
-         ClientProducer producer4 = sessSend.createProducer(ADDRESS);
-
-         for (int i = 0; i < numMessages; i++)
-         {
-            ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
-                                                                false,
-                                                                0,
-                                                                System.currentTimeMillis(),
-                                                                (byte)1);
-            message.putIntProperty(new SimpleString("count"), i);
-            message.getBody().putString("aardvarks");
-            message.getBody().flip();
-            producer4.send(message);
-         }
-         
-         
-         class MyHandler implements MessageHandler
-         {
-            final CountDownLatch latch = new CountDownLatch(1);
-
-            volatile int count;
-
-            public void onMessage(ClientMessage message)
-            {
-               //log.info("got message " + message.getMessageID());
-               
-               assertEquals(count, message.getProperty(new SimpleString("count")));
-
-               //log.info(this + " got message " + count);
-
-               count++;
-
-               if (count == numMessages)
-               {
-                  latch.countDown();
-               }
-            }
-         }
-
-         Set<MyHandler> handlers = new HashSet<MyHandler>();
-
-         for (ClientConsumer consumer : consumers)
-         {
-            MyHandler handler = new MyHandler();
-
-            consumer.setMessageHandler(handler);
-
-            handlers.add(handler);
-         }
-
-         for (MyHandler handler : handlers)
-         {
-            boolean ok = handler.latch.await(1000, TimeUnit.MILLISECONDS);
-
-            assertTrue(ok);
-         }
-
-         sessSend.close();
-         sessConsume.close();
-
-         long end = System.currentTimeMillis();
-
-         log.info("duration " + (end - start));
-
-         // this.waitForFailThread();
-
-         stop();
-      }
-   }
-
-   // public void testHangOnCreateSession(final byte type) throws Exception
-   // {
-   // for (int j = 0; j < 1000; j++)
-   // {
-   // start();
-   //
-   // ClientSessionFactory sf = new ClientSessionFactoryImpl(new
-   // TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"),
-   // new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
-   // backupParams));
-   //
-   // final int numSessions = 50;
-   //
-   // List<ClientSession> sessions = new ArrayList<ClientSession>();
-   //
-   // for (int i = 0; i < numSessions; i++)
-   // {
-   // ClientSession session = sf.createSession(false, true, true, -1, false);
-   //
-   // sessions.add(session);
-   //
-   // if (i == 0)
-   // {
-   // RemotingConnectionImpl conn = (RemotingConnectionImpl)((ClientSessionImpl)session).getConnection();
-   //
-   // //conn.failOnPacketType(type);
-   // }
-   // }
-   //
-   // for (ClientSession session : sessions)
-   // {
-   // session.close();
-   // }
-   //
-   // stop();
-   // }
-   //
-   // }
-
-   // Package protected ---------------------------------------------
-
-   // Protected -----------------------------------------------------
-
-   private void start() throws Exception
-   {
-      Configuration backupConf = new ConfigurationImpl();
-      backupConf.setSecurityEnabled(false);
-      backupConf.setPacketConfirmationBatchSize(10);
-      backupParams.put(TransportConstants.SERVER_ID_PROP_NAME, 1);
-      backupConf.getAcceptorConfigurations()
-                .add(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMAcceptorFactory",
-                                                backupParams));
-      backupConf.setBackup(true);
-      backupService = MessagingServiceImpl.newNullStorageMessagingServer(backupConf);
-      backupService.start();
-
-      Configuration liveConf = new ConfigurationImpl();
-      liveConf.setSecurityEnabled(false);
-      liveConf.setPacketConfirmationBatchSize(10);
-      liveConf.getAcceptorConfigurations()
-              .add(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMAcceptorFactory"));
-      liveConf.setBackupConnectorConfiguration(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
-                                                                          backupParams));
-      liveService = MessagingServiceImpl.newNullStorageMessagingServer(liveConf);
-      liveService.start();
-   }
-
-   private void stop() throws Exception
-   {
-      ConnectionRegistryImpl.instance.dump();
-
-      assertEquals(0, ConnectionRegistryImpl.instance.size());
-
-      assertEquals(0, backupService.getServer().getRemotingService().getConnections().size());
-
-      backupService.stop();
-
-      assertEquals(0, liveService.getServer().getRemotingService().getConnections().size());
-
-      liveService.stop();
-
-      assertEquals(0, InVMRegistry.instance.size());
-   }
-
-   // Private -------------------------------------------------------
-
-   // Inner classes -------------------------------------------------
-}

Added: trunk/tests/src/org/jboss/messaging/tests/soak/failover/RandomFailoverSoakTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/soak/failover/RandomFailoverSoakTest.java	                        (rev 0)
+++ trunk/tests/src/org/jboss/messaging/tests/soak/failover/RandomFailoverSoakTest.java	2008-10-16 19:14:11 UTC (rev 5123)
@@ -0,0 +1,45 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2008, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+
+package org.jboss.messaging.tests.soak.failover;
+
+import org.jboss.messaging.tests.integration.cluster.RandomFailoverTest;
+
+/**
+ * A RandomFailoverSoakTest
+ *
+ * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
+ * 
+ * Created 9 Oct 2008 17:33:05
+ *
+ *
+ */
+public class RandomFailoverSoakTest extends RandomFailoverTest
+{
+
+   protected int getNumIterations()
+   {
+      return 500;
+   }
+  
+}




More information about the jboss-cvs-commits mailing list