[jboss-cvs] JBoss Messaging SVN: r7536 - in branches/Branch_MultiThreaded_Replication: src/main/org/jboss/messaging/core/management and 17 other directories.

jboss-cvs-commits at lists.jboss.org jboss-cvs-commits at lists.jboss.org
Tue Jul 7 15:45:09 EDT 2009


Author: timfox
Date: 2009-07-07 15:45:08 -0400 (Tue, 07 Jul 2009)
New Revision: 7536

Added:
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareSharedCounter.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/SequencedLock.java
   branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/unit/core/server/replication/impl/SequencedLockTest.java
Removed:
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/PriorityLock.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareAtomicLong.java
   branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/unit/core/server/replication/impl/NewSequencedLockTest.java
Modified:
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ClientSessionImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ConnectionManager.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ConnectionManagerImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/management/NotificationService.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/management/impl/ManagementServiceImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalStorageManager.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/persistence/impl/nullpm/NullStorageManager.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/ClusterQueueStateManager.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/ClusterQueueStateManagerImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/SimpleAddressManager.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/Channel.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/ChannelImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/PacketDecoder.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/RemotingConnectionImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/invm/InVMRegistry.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/wireformat/PacketImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/server/impl/RemotingServiceImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/MessagingServer.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/Queue.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/QueueFactory.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/ServerSession.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/cluster/impl/BridgeImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/cluster/impl/Redistributor.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/LastValueQueue.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/MessagingServerPacketHandler.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueFactoryImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueReplicationPacketHandler.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerSessionPacketHandler.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/Replicator.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareReadWriteLock.java
   branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicatorImpl.java
   branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadFailoverSupport.java
   branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/integration/cluster/failover/SimpleAutomaticFailoverTest.java
Log:
MT replication

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ClientSessionImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ClientSessionImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ClientSessionImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -733,8 +733,6 @@
 
          ReattachSessionResponseMessage response = (ReattachSessionResponseMessage)channel1.sendBlocking(request);
          
-         log.info("Got a response with a last received command id " + response.getLastConfirmedCommandID());
-
          if (!response.isRemoved())
          {
             channel.replayCommands(response.getLastConfirmedCommandID());

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ConnectionManager.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ConnectionManager.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ConnectionManager.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -64,5 +64,7 @@
    
    RemotingConnection getConnection(final int initialRefCount);
    
+   void returnConnection(RemotingConnection conn);
+   
    void close();
 }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ConnectionManagerImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ConnectionManagerImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/client/impl/ConnectionManagerImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -469,10 +469,37 @@
             checkCloseConnections();
          }
       }
-      
+
       closed = true;
    }
+   
+   public void returnConnection(final RemotingConnection conn)
+   {
+      synchronized (createSessionLock)
+      {
+         synchronized (failoverLock)
+         {
+            connections.remove(conn.getID());
 
+            if (refCount != 0)
+            {
+               refCount--;
+            }
+            
+            pingers.remove(conn.getID());
+
+            try
+            {
+               conn.destroy();
+            }
+            catch (Throwable ignore)
+            {
+            }                       
+         }
+      }
+
+   }
+
    // Public
    // ---------------------------------------------------------------------------------------
 
@@ -866,7 +893,7 @@
    public RemotingConnection getConnection(final int initialRefCount)
    {
       RemotingConnection conn;
-
+      
       if (connections.size() < maxConnections)
       {
          // Create a new one

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/management/NotificationService.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/management/NotificationService.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/management/NotificationService.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -49,8 +49,8 @@
 
    void enableNotifications(boolean enable);
 
-   void addNotificationListener(NotificationListener listener);
+   //void addNotificationListener(NotificationListener listener);
 
-   void removeNotificationListener(NotificationListener listener);
+  // void removeNotificationListener(NotificationListener listener);
 
 }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/management/impl/ManagementServiceImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/management/impl/ManagementServiceImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/management/impl/ManagementServiceImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -56,7 +56,6 @@
 import org.jboss.messaging.core.management.DivertControl;
 import org.jboss.messaging.core.management.ManagementService;
 import org.jboss.messaging.core.management.Notification;
-import org.jboss.messaging.core.management.NotificationListener;
 import org.jboss.messaging.core.management.ObjectNames;
 import org.jboss.messaging.core.management.ReplicationOperationInvoker;
 import org.jboss.messaging.core.management.ResourceNames;
@@ -86,7 +85,6 @@
 import org.jboss.messaging.core.settings.impl.AddressSettings;
 import org.jboss.messaging.core.transaction.ResourceManager;
 import org.jboss.messaging.utils.SimpleString;
-import org.jboss.messaging.utils.TypedProperties;
 
 /*
  * @author <a href="mailto:jmesnil at redhat.com">Jeff Mesnil</a>
@@ -117,9 +115,9 @@
    private HierarchicalRepository<AddressSettings> addressSettingsRepository;
 
    private MessagingServerControlImpl messagingServerControl;
-   
-   private MessagingServer messagingServer;
 
+  // private MessagingServer messagingServer;
+
    private final MessageCounterManager messageCounterManager;
 
    private final SimpleString managementNotificationAddress;
@@ -138,10 +136,10 @@
 
    private boolean notificationsEnabled;
 
-   private final Set<NotificationListener> listeners = new org.jboss.messaging.utils.ConcurrentHashSet<NotificationListener>();
+ //  private final Set<NotificationListener> listeners = new org.jboss.messaging.utils.ConcurrentHashSet<NotificationListener>();
 
    private ReplicationOperationInvoker replicationInvoker;
-   
+
    private ClusterQueueStateManager clusterQueueStateManager;
 
    // Static --------------------------------------------------------
@@ -157,7 +155,9 @@
 
    // Constructor ----------------------------------------------------
 
-   public ManagementServiceImpl(final MBeanServer mbeanServer, final Configuration configuration, final int managementConnectorID)
+   public ManagementServiceImpl(final MBeanServer mbeanServer,
+                                final Configuration configuration,
+                                final int managementConnectorID)
    {
       this.mbeanServer = mbeanServer;
       this.jmxManagementEnabled = configuration.isJMXManagementEnabled();
@@ -210,7 +210,7 @@
       this.securityRepository = securityRepository;
       this.storageManager = storageManager;
       this.clusterQueueStateManager = clusterQueueStateManager;
-      this.messagingServer = messagingServer;
+    //  this.messagingServer = messagingServer;
 
       JBMSecurityManager sm = messagingServer.getSecurityManager();
       if (sm != null)
@@ -522,16 +522,16 @@
       }
    }
 
-   public void addNotificationListener(final NotificationListener listener)
-   {
-      listeners.add(listener);
-   }
+//   public void addNotificationListener(final NotificationListener listener)
+//   {
+//      listeners.add(listener);
+//   }
+//
+//   public void removeNotificationListener(final NotificationListener listener)
+//   {
+//      listeners.remove(listener);
+//   }
 
-   public void removeNotificationListener(final NotificationListener listener)
-   {
-      listeners.remove(listener);
-   }
-
    public SimpleString getManagementAddress()
    {
       return managementAddress;
@@ -615,69 +615,7 @@
    {
       if (messagingServerControl != null && notificationsEnabled)
       {
-         // This needs to be synchronized since we need to ensure notifications are processed in strict sequence
-         synchronized (this)
-         {
-            // We also need to synchronize on the post office notification lock
-            // otherwise we can get notifications arriving in wrong order / missing
-            // if a notification occurs at same time as sendQueueInfoToQueue is processed
-            synchronized (clusterQueueStateManager.getNotificationLock())
-            {
-
-               // First send to any local listeners
-               for (NotificationListener listener : listeners)
-               {
-                  try
-                  {
-                     listener.onNotification(notification);
-                  }
-                  catch (Exception e)
-                  {
-                     // Exception thrown from one listener should not stop execution of others
-                     log.error("Failed to call listener", e);
-                  }
-               }
-
-               // start sending notification *messages* only when the server if fully started
-               if (messagingServer != null && !messagingServer.isStarted())
-               {
-                  return;
-               }
-
-               ServerMessage notificationMessage = new ServerMessageImpl(storageManager.generateUniqueID());
-
-               notificationMessage.setBody(ChannelBuffers.EMPTY_BUFFER);
-               // Notification messages are always durable so the user can choose whether to add a durable queue to
-               // consume
-               // them in
-               notificationMessage.setDurable(true);
-               notificationMessage.setDestination(managementNotificationAddress);
-
-               TypedProperties notifProps;
-               if (notification.getProperties() != null)
-               {
-                  notifProps = new TypedProperties(notification.getProperties());
-               }
-               else
-               {
-                  notifProps = new TypedProperties();
-               }
-
-               notifProps.putStringProperty(ManagementHelper.HDR_NOTIFICATION_TYPE,
-                                            new SimpleString(notification.getType().toString()));
-
-               notifProps.putLongProperty(ManagementHelper.HDR_NOTIFICATION_TIMESTAMP, System.currentTimeMillis());
-
-               if (notification.getUID() != null)
-               {
-                  notifProps.putStringProperty(new SimpleString("foobar"), new SimpleString(notification.getUID()));
-               }
-
-               notificationMessage.putTypedProperties(notifProps);
-
-               postOffice.route(notificationMessage, null);
-            }
-         }
+         clusterQueueStateManager.sendNotification(notification, managementNotificationAddress);
       }
    }
 
@@ -801,6 +739,5 @@
       return result;
    }
 
-
    // Inner classes -------------------------------------------------
 }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalStorageManager.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalStorageManager.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalStorageManager.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -68,7 +68,7 @@
 import org.jboss.messaging.core.server.Queue;
 import org.jboss.messaging.core.server.ServerMessage;
 import org.jboss.messaging.core.server.impl.ServerMessageImpl;
-import org.jboss.messaging.core.server.replication.impl.ReplicationAwareAtomicLong;
+import org.jboss.messaging.core.server.replication.impl.ReplicationAwareSharedCounter;
 import org.jboss.messaging.core.transaction.ResourceManager;
 import org.jboss.messaging.core.transaction.Transaction;
 import org.jboss.messaging.core.transaction.TransactionOperation;
@@ -1114,7 +1114,7 @@
 
    private class BatchingIDGenerator implements IDGenerator
    {
-      private final ReplicationAwareAtomicLong counter;
+      private final ReplicationAwareSharedCounter counter;
 
       private final long checkpointSize;
 
@@ -1122,7 +1122,7 @@
 
       public BatchingIDGenerator(final long start, final long checkpointSize)
       {
-         this.counter = new ReplicationAwareAtomicLong(start);
+         this.counter = new ReplicationAwareSharedCounter(backup, start);
 
          this.checkpointSize = checkpointSize;
 

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/persistence/impl/nullpm/NullStorageManager.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/persistence/impl/nullpm/NullStorageManager.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/persistence/impl/nullpm/NullStorageManager.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -37,7 +37,7 @@
 import org.jboss.messaging.core.server.MessageReference;
 import org.jboss.messaging.core.server.Queue;
 import org.jboss.messaging.core.server.ServerMessage;
-import org.jboss.messaging.core.server.replication.impl.ReplicationAwareAtomicLong;
+import org.jboss.messaging.core.server.replication.impl.ReplicationAwareSharedCounter;
 import org.jboss.messaging.core.transaction.ResourceManager;
 import org.jboss.messaging.utils.Pair;
 import org.jboss.messaging.utils.SimpleString;
@@ -55,7 +55,7 @@
 {
    private static final Logger log = Logger.getLogger(NullStorageManager.class);
    
-   private final ReplicationAwareAtomicLong idSequence = new ReplicationAwareAtomicLong(0);
+   private final ReplicationAwareSharedCounter idSequence;
    
    private UUID id;
 
@@ -66,6 +66,8 @@
    public NullStorageManager(final boolean backup)
    {
       this.backup = backup;
+      
+      idSequence = new ReplicationAwareSharedCounter(backup, 0);
    }
    
    public UUID getPersistentID()

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/ClusterQueueStateManager.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/ClusterQueueStateManager.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/ClusterQueueStateManager.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -23,6 +23,7 @@
 
 package org.jboss.messaging.core.postoffice;
 
+import org.jboss.messaging.core.management.Notification;
 import org.jboss.messaging.core.server.MessagingComponent;
 import org.jboss.messaging.utils.SimpleString;
 
@@ -35,7 +36,7 @@
  */
 public interface ClusterQueueStateManager extends MessagingComponent
 {
-   Object getNotificationLock();
+   void sendQueueInfoToQueue(SimpleString queueName, SimpleString address) throws Exception;
    
-   void sendQueueInfoToQueue(SimpleString queueName, SimpleString address) throws Exception;
+   void sendNotification(Notification notification, SimpleString dest) throws Exception;
 }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/ClusterQueueStateManagerImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/ClusterQueueStateManagerImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/ClusterQueueStateManagerImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -26,12 +26,13 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
 
 import org.jboss.messaging.core.buffers.ChannelBuffers;
 import org.jboss.messaging.core.client.management.impl.ManagementHelper;
-import org.jboss.messaging.core.management.ManagementService;
 import org.jboss.messaging.core.management.Notification;
-import org.jboss.messaging.core.management.NotificationListener;
 import org.jboss.messaging.core.management.NotificationType;
 import org.jboss.messaging.core.persistence.StorageManager;
 import org.jboss.messaging.core.postoffice.Binding;
@@ -39,9 +40,11 @@
 import org.jboss.messaging.core.postoffice.ClusterQueueStateManager;
 import org.jboss.messaging.core.postoffice.PostOffice;
 import org.jboss.messaging.core.postoffice.QueueInfo;
+import org.jboss.messaging.core.server.MessagingServer;
 import org.jboss.messaging.core.server.Queue;
 import org.jboss.messaging.core.server.ServerMessage;
 import org.jboss.messaging.core.server.impl.ServerMessageImpl;
+import org.jboss.messaging.core.server.replication.impl.ReplicationAwareReadWriteLock;
 import org.jboss.messaging.core.settings.HierarchicalRepository;
 import org.jboss.messaging.core.settings.impl.AddressSettings;
 import org.jboss.messaging.utils.ExecutorFactory;
@@ -56,7 +59,7 @@
  *
  *
  */
-public class ClusterQueueStateManagerImpl implements ClusterQueueStateManager, NotificationListener
+public class ClusterQueueStateManagerImpl implements ClusterQueueStateManager
 {
    public static final SimpleString HDR_RESET_QUEUE_DATA = new SimpleString("_JBM_RESET_QUEUE_DATA");
 
@@ -66,29 +69,31 @@
 
    private final StorageManager storageManager;
 
-   private final ExecutorFactory redistributorExecutorFactory;
+   private final ExecutorService threadPool;
 
    private final HierarchicalRepository<AddressSettings> addressSettingsRepository;
 
-   private final ManagementService managementService;
+   private final Lock lock;
 
    private boolean started;
 
-   public ClusterQueueStateManagerImpl(final ExecutorFactory redistributorExecutorFactory,
-                                       final HierarchicalRepository<AddressSettings> addressSettingsRepository,
-                                       final PostOffice postOffice,
-                                       final StorageManager storageManager,
-                                       final ManagementService managementService)
+   private final MessagingServer server;
+
+   public ClusterQueueStateManagerImpl(final MessagingServer server)
    {
-      this.redistributorExecutorFactory = redistributorExecutorFactory;
+      this.threadPool = server.getThreadPool();
 
-      this.addressSettingsRepository = addressSettingsRepository;
+      this.addressSettingsRepository = server.getAddressSettingsRepository();
 
-      this.postOffice = postOffice;
+      this.postOffice = server.getPostOffice();
 
-      this.storageManager = storageManager;
+      this.storageManager = server.getStorageManager();
 
-      this.managementService = managementService;
+      this.server = server;
+
+      ReadWriteLock rwLock = new ReplicationAwareReadWriteLock("clusterqueuestatemanager", 0, true);
+
+      lock = rwLock.writeLock();
    }
 
    public synchronized void start()
@@ -98,8 +103,6 @@
          return;
       }
 
-      managementService.addNotificationListener(this);
-
       started = true;
    }
 
@@ -110,8 +113,6 @@
          return;
       }
 
-      managementService.removeNotificationListener(this);
-
       started = false;
    }
 
@@ -120,13 +121,177 @@
       return started;
    }
 
-   public Object getNotificationLock()
+   public Lock getNotificationLock()
    {
-      return this;
+      return lock;
    }
 
-   public synchronized void onNotification(final Notification notification)
+   public void sendNotification(final Notification notification, final SimpleString dest) throws Exception
    {
+      ServerMessage notificationMessage;
+
+      lock.lock();
+
+      try
+      {
+         handleNotification(notification);
+
+         // start sending notification *messages* only when the server if fully started
+         if (server != null && !server.isStarted())
+         {
+            return;
+         }
+
+         notificationMessage = new ServerMessageImpl(storageManager.generateUniqueID());
+
+         notificationMessage.setBody(ChannelBuffers.EMPTY_BUFFER);
+         // Notification messages are always durable so the user can choose whether to add a durable queue to
+         // consume them in
+         notificationMessage.setDurable(true);
+         notificationMessage.setDestination(dest);
+
+         TypedProperties notifProps;
+         if (notification.getProperties() != null)
+         {
+            notifProps = new TypedProperties(notification.getProperties());
+         }
+         else
+         {
+            notifProps = new TypedProperties();
+         }
+
+         notifProps.putStringProperty(ManagementHelper.HDR_NOTIFICATION_TYPE, new SimpleString(notification.getType()
+                                                                                                           .toString()));
+
+         notifProps.putLongProperty(ManagementHelper.HDR_NOTIFICATION_TIMESTAMP, System.currentTimeMillis());
+
+         if (notification.getUID() != null)
+         {
+            notifProps.putStringProperty(new SimpleString("foobar"), new SimpleString(notification.getUID()));
+         }
+
+         notificationMessage.putTypedProperties(notifProps);
+      }
+      finally
+      {
+         lock.unlock();
+      }
+
+      // Needs to be outside the lock - otherwise can get deadlock when replicating as one thread has this monitor
+      // then waits getting the priority lock
+      // on the postoffice, but waits for ever since the thread it's waiting for is blocked on this monitor
+      postOffice.route(notificationMessage, null);
+   }
+
+   public void sendQueueInfoToQueue(final SimpleString queueName, final SimpleString address) throws Exception
+   {
+      // We send direct to the queue so we can send it to the same queue that is bound to the notifications adress -
+      // this is crucial for ensuring
+      // that queue infos and notifications are received in a contiguous consistent stream
+      Binding binding = postOffice.getBinding(queueName);
+
+      if (binding == null)
+      {
+         throw new IllegalStateException("Cannot find queue " + queueName);
+      }
+
+      Queue queue = (Queue)binding.getBindable();
+
+      // Need to lock to make sure all queue info and notifications are in the correct order with no gaps
+      lock.lock();
+      try
+      {
+         // First send a reset message
+
+         ServerMessage message = new ServerMessageImpl(storageManager.generateUniqueID());
+         message.setBody(ChannelBuffers.EMPTY_BUFFER);
+         message.setDestination(queueName);
+         message.putBooleanProperty(HDR_RESET_QUEUE_DATA, true);
+         queue.preroute(message, null);
+         queue.route(message, null);
+
+         for (QueueInfo info : queueInfos.values())
+         {
+            if (info.getAddress().startsWith(address))
+            {
+               message = createQueueInfoMessage(NotificationType.BINDING_ADDED, queueName);
+
+               message.putStringProperty(ManagementHelper.HDR_ADDRESS, info.getAddress());
+               message.putStringProperty(ManagementHelper.HDR_CLUSTER_NAME, info.getClusterName());
+               message.putStringProperty(ManagementHelper.HDR_ROUTING_NAME, info.getRoutingName());
+               message.putLongProperty(ManagementHelper.HDR_BINDING_ID, info.getID());
+               message.putStringProperty(ManagementHelper.HDR_FILTERSTRING, info.getFilterString());
+               message.putIntProperty(ManagementHelper.HDR_DISTANCE, info.getDistance());
+
+               routeDirect(queue, message);
+
+               int consumersWithFilters = info.getFilterStrings() != null ? info.getFilterStrings().size() : 0;
+
+               for (int i = 0; i < info.getNumberOfConsumers() - consumersWithFilters; i++)
+               {
+                  message = createQueueInfoMessage(NotificationType.CONSUMER_CREATED, queueName);
+
+                  message.putStringProperty(ManagementHelper.HDR_ADDRESS, info.getAddress());
+                  message.putStringProperty(ManagementHelper.HDR_CLUSTER_NAME, info.getClusterName());
+                  message.putStringProperty(ManagementHelper.HDR_ROUTING_NAME, info.getRoutingName());
+                  message.putIntProperty(ManagementHelper.HDR_DISTANCE, info.getDistance());
+
+                  routeDirect(queue, message);
+               }
+
+               if (info.getFilterStrings() != null)
+               {
+                  for (SimpleString filterString : info.getFilterStrings())
+                  {
+                     message = createQueueInfoMessage(NotificationType.CONSUMER_CREATED, queueName);
+
+                     message.putStringProperty(ManagementHelper.HDR_ADDRESS, info.getAddress());
+                     message.putStringProperty(ManagementHelper.HDR_CLUSTER_NAME, info.getClusterName());
+                     message.putStringProperty(ManagementHelper.HDR_ROUTING_NAME, info.getRoutingName());
+                     message.putStringProperty(ManagementHelper.HDR_FILTERSTRING, filterString);
+                     message.putIntProperty(ManagementHelper.HDR_DISTANCE, info.getDistance());
+
+                     routeDirect(queue, message);
+                  }
+               }
+            }
+         }
+      }
+      finally
+      {
+         lock.unlock();
+      }
+
+   }
+
+   private void routeDirect(final Queue queue, final ServerMessage message) throws Exception
+   {
+      if (queue.getFilter() == null || queue.getFilter().match(message))
+      {
+         queue.preroute(message, null);
+         queue.route(message, null);
+      }
+   }
+
+   private ServerMessage createQueueInfoMessage(final NotificationType type, final SimpleString queueName)
+   {
+      ServerMessage message = new ServerMessageImpl(storageManager.generateUniqueID());
+      message.setBody(ChannelBuffers.EMPTY_BUFFER);
+
+      message.setDestination(queueName);
+
+      String uid = UUIDGenerator.getInstance().generateStringUUID();
+
+      message.putStringProperty(ManagementHelper.HDR_NOTIFICATION_TYPE, new SimpleString(type.toString()));
+      message.putLongProperty(ManagementHelper.HDR_NOTIFICATION_TIMESTAMP, System.currentTimeMillis());
+
+      message.putStringProperty(new SimpleString("foobar"), new SimpleString(uid));
+
+      return message;
+   }
+
+   private void handleNotification(final Notification notification)
+   {
       NotificationType type = notification.getType();
 
       switch (type)
@@ -251,7 +416,7 @@
 
                   if (redistributionDelay != -1)
                   {
-                     queue.addRedistributor(redistributionDelay, redistributorExecutorFactory.getExecutor());
+                     queue.addRedistributor(redistributionDelay, threadPool);
                   }
                }
             }
@@ -320,7 +485,7 @@
 
                   if (redistributionDelay != -1)
                   {
-                     queue.addRedistributor(redistributionDelay, redistributorExecutorFactory.getExecutor());
+                     queue.addRedistributor(redistributionDelay, threadPool);
                   }
                }
             }
@@ -337,106 +502,4 @@
 
       }
    }
-
-   public void sendQueueInfoToQueue(final SimpleString queueName, final SimpleString address) throws Exception
-   {
-      // We send direct to the queue so we can send it to the same queue that is bound to the notifications adress -
-      // this is crucial for ensuring
-      // that queue infos and notifications are received in a contiguous consistent stream
-      Binding binding = postOffice.getBinding(queueName);
-
-      if (binding == null)
-      {
-         throw new IllegalStateException("Cannot find queue " + queueName);
-      }
-
-      Queue queue = (Queue)binding.getBindable();
-
-      // Need to lock to make sure all queue info and notifications are in the correct order with no gaps
-      synchronized (this)
-      {
-         // First send a reset message
-
-         ServerMessage message = new ServerMessageImpl(storageManager.generateUniqueID());
-         message.setBody(ChannelBuffers.EMPTY_BUFFER);
-         message.setDestination(queueName);
-         message.putBooleanProperty(HDR_RESET_QUEUE_DATA, true);
-         queue.preroute(message, null);
-         queue.route(message, null);
-
-         for (QueueInfo info : queueInfos.values())
-         {
-            if (info.getAddress().startsWith(address))
-            {
-               message = createQueueInfoMessage(NotificationType.BINDING_ADDED, queueName);
-
-               message.putStringProperty(ManagementHelper.HDR_ADDRESS, info.getAddress());
-               message.putStringProperty(ManagementHelper.HDR_CLUSTER_NAME, info.getClusterName());
-               message.putStringProperty(ManagementHelper.HDR_ROUTING_NAME, info.getRoutingName());
-               message.putLongProperty(ManagementHelper.HDR_BINDING_ID, info.getID());
-               message.putStringProperty(ManagementHelper.HDR_FILTERSTRING, info.getFilterString());
-               message.putIntProperty(ManagementHelper.HDR_DISTANCE, info.getDistance());
-
-               routeDirect(queue, message);
-
-               int consumersWithFilters = info.getFilterStrings() != null ? info.getFilterStrings().size() : 0;
-
-               for (int i = 0; i < info.getNumberOfConsumers() - consumersWithFilters; i++)
-               {
-                  message = createQueueInfoMessage(NotificationType.CONSUMER_CREATED, queueName);
-
-                  message.putStringProperty(ManagementHelper.HDR_ADDRESS, info.getAddress());
-                  message.putStringProperty(ManagementHelper.HDR_CLUSTER_NAME, info.getClusterName());
-                  message.putStringProperty(ManagementHelper.HDR_ROUTING_NAME, info.getRoutingName());
-                  message.putIntProperty(ManagementHelper.HDR_DISTANCE, info.getDistance());
-
-                  routeDirect(queue, message);
-               }
-
-               if (info.getFilterStrings() != null)
-               {
-                  for (SimpleString filterString : info.getFilterStrings())
-                  {
-                     message = createQueueInfoMessage(NotificationType.CONSUMER_CREATED, queueName);
-
-                     message.putStringProperty(ManagementHelper.HDR_ADDRESS, info.getAddress());
-                     message.putStringProperty(ManagementHelper.HDR_CLUSTER_NAME, info.getClusterName());
-                     message.putStringProperty(ManagementHelper.HDR_ROUTING_NAME, info.getRoutingName());
-                     message.putStringProperty(ManagementHelper.HDR_FILTERSTRING, filterString);
-                     message.putIntProperty(ManagementHelper.HDR_DISTANCE, info.getDistance());
-
-                     routeDirect(queue, message);
-                  }
-               }
-            }
-         }
-      }
-
-   }
-
-   private void routeDirect(final Queue queue, final ServerMessage message) throws Exception
-   {
-      if (queue.getFilter() == null || queue.getFilter().match(message))
-      {
-         queue.preroute(message, null);
-         queue.route(message, null);
-      }
-   }
-
-   private ServerMessage createQueueInfoMessage(final NotificationType type, final SimpleString queueName)
-   {
-      ServerMessage message = new ServerMessageImpl(storageManager.generateUniqueID());
-      message.setBody(ChannelBuffers.EMPTY_BUFFER);
-
-      message.setDestination(queueName);
-
-      String uid = UUIDGenerator.getInstance().generateStringUUID();
-
-      message.putStringProperty(ManagementHelper.HDR_NOTIFICATION_TYPE, new SimpleString(type.toString()));
-      message.putLongProperty(ManagementHelper.HDR_NOTIFICATION_TIMESTAMP, System.currentTimeMillis());
-
-      message.putStringProperty(new SimpleString("foobar"), new SimpleString(uid));
-
-      return message;
-   }
 }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -178,7 +178,7 @@
 
       // this.addressSettingsRepository = addressSettingsRepository;
       
-      lock = new ReplicationAwareReadWriteLock("postoffice", 0);
+      lock = new ReplicationAwareReadWriteLock("postoffice", 0, true);
    }
 
    // MessagingComponent implementation ---------------------------------------
@@ -457,37 +457,38 @@
    // even though failover is complete
    public void addBinding(final Binding binding) throws Exception
    {
+      boolean addressExists;
       lock.writeLock().lock();
       try
       {
-         //binding.setID(generateTransientID());
+         addressExists = addressManager.getBindingsForRoutingAddress(binding.getAddress()) != null;
 
-         boolean addressExists = addressManager.getBindingsForRoutingAddress(binding.getAddress()) != null;
-
          addressManager.addBinding(binding);
+         
+      }
+      finally
+      {
+         lock.writeLock().unlock();
+      }
 
-         if (binding.getType() == BindingType.LOCAL_QUEUE)
+      if (binding.getType() == BindingType.LOCAL_QUEUE)
+      {
+         Queue queue = (Queue)binding.getBindable();
+
+         if (backup)
          {
-            Queue queue = (Queue)binding.getBindable();
+            queue.setBackup();
+         }
 
-            if (backup)
-            {
-               queue.setBackup();
-            }
+         managementService.registerQueue(queue, binding.getAddress(), storageManager);
 
-            managementService.registerQueue(queue, binding.getAddress(), storageManager);
-
-            if (!addressExists)
-            {
-               managementService.registerAddress(binding.getAddress());
-            }
+         if (!addressExists)
+         {
+            managementService.registerAddress(binding.getAddress());
          }
       }
-      finally
-      {
-         lock.writeLock().unlock();
-      }
       
+      
       TypedProperties props = new TypedProperties();
 
       props.putIntProperty(ManagementHelper.HDR_BINDING_TYPE, binding.getType().toInt());
@@ -521,37 +522,35 @@
       try
       {
          binding = addressManager.removeBinding(uniqueName);
-         if (binding == null)
-         {
-            throw new MessagingException(MessagingException.QUEUE_DOES_NOT_EXIST);
-         }
+      }
+      finally
+      {
+         lock.writeLock().unlock();
+      }
+      if (binding == null)
+      {
+         throw new MessagingException(MessagingException.QUEUE_DOES_NOT_EXIST);
+      }
 
-         if (binding.getType() == BindingType.LOCAL_QUEUE)
-         {
-            managementService.unregisterQueue(uniqueName, binding.getAddress());
+      if (binding.getType() == BindingType.LOCAL_QUEUE)
+      {
+         managementService.unregisterQueue(uniqueName, binding.getAddress());
 
-            if (addressManager.getBindingsForRoutingAddress(binding.getAddress()) == null)
-            {
-               managementService.unregisterAddress(binding.getAddress());
-            }
-         }
-         else if (binding.getType() == BindingType.DIVERT)
+         if (addressManager.getBindingsForRoutingAddress(binding.getAddress()) == null)
          {
-            managementService.unregisterDivert(uniqueName);
-
-            if (addressManager.getBindingsForRoutingAddress(binding.getAddress()) == null)
-            {
-               managementService.unregisterAddress(binding.getAddress());
-            }
+            managementService.unregisterAddress(binding.getAddress());
          }
-         
-         //releaseTransientID(binding.getID());         
       }
-      finally
+      else if (binding.getType() == BindingType.DIVERT)
       {
-         lock.writeLock().unlock();
-      }
-      
+         managementService.unregisterDivert(uniqueName);
+
+         if (addressManager.getBindingsForRoutingAddress(binding.getAddress()) == null)
+         {
+            managementService.unregisterAddress(binding.getAddress());
+         }
+      }      
+            
       TypedProperties props = new TypedProperties();
 
       props.putStringProperty(ManagementHelper.HDR_ADDRESS, binding.getAddress());

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/SimpleAddressManager.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/SimpleAddressManager.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/postoffice/impl/SimpleAddressManager.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -59,7 +59,7 @@
       
       idMap.put(binding.getID(), binding);
       
-      addMappingInternal(binding.getAddress(), binding);          
+      addMappingInternal(binding.getAddress(), binding);    
    }
 
    public Binding removeBinding(final SimpleString uniqueName)

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/Channel.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/Channel.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/Channel.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -67,5 +67,5 @@
    
    //void waitForAllReplicationResponse();
    
-   void replicationResponseReceived(Replicator replicator);
+   //void replicationResponseReceived(Replicator replicator, int count);
 }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/ChannelImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/ChannelImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/ChannelImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -25,7 +25,6 @@
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.EARLY_RESPONSE;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.PACKETS_CONFIRMED;
 
-import java.util.Iterator;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
@@ -44,8 +43,6 @@
 import org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl;
 import org.jboss.messaging.core.remoting.impl.wireformat.PacketsConfirmedMessage;
 import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
-import org.jboss.messaging.core.server.replication.Replicator;
-import org.jboss.messaging.core.server.replication.impl.JBMThread;
 
 /**
  * A ChannelImpl
@@ -84,12 +81,8 @@
 
    private final Object sendBlockingLock = new Object();
 
-   // private final Object replicationLock = new Object();
-
    private boolean failingOver;
 
-   // private final Queue<Runnable> responseActions = new ConcurrentLinkedQueue<Runnable>();
-
    private final int windowSize;
 
    private final int confWindowSize;
@@ -100,10 +93,6 @@
 
    private CommandConfirmationHandler commandConfirmationHandler;
 
-   // private int responseActionCount;
-
-   // private boolean playedResponsesOnFailure;
-
    public ChannelImpl(final RemotingConnection connection, final long id, final int windowSize, final boolean block)
    {
       this.connection = connection;
@@ -176,30 +165,13 @@
       send(packet, false);
    }
 
-   // This must never called by more than one thread concurrently
+   private final QueuedWriteManager queuedWriteManager = new QueuedWriteManager(this);
+
    public void send(final Packet packet, final boolean flush)
    {
-      // FIXME - this is a bit hacky
-
-      Thread t = Thread.currentThread();
-
-      if (t instanceof JBMThread)
+      if (queuedWriteManager.tryQueue(packet))
       {
-         JBMThread thread = (JBMThread)t;
-
-         if (thread.isRecording())
-         {
-            thread.getReplicator().registerWaitingChannel(this);
-
-            QueuedWrite qw = new QueuedWrite();
-            qw.replicator = thread.getReplicator();
-            qw.packet = packet;
-            //qw.sequence = qw.replicator.getReplicateSequence();
-
-            queuedWrites.add(qw);
-
-            return;
-         }
+         return;
       }
 
       synchronized (sendLock)
@@ -368,100 +340,11 @@
       }
    }
 
-   // Must be synchronized since can be called by incoming session commands but also by deliveries
-   // Also needs to be synchronized with respect to replicatingChannelDead
-   // public void replicatePacket(final Packet packet, final long replicatedChannelID, final Runnable action)
-   // {
-   // packet.setChannelID(replicatedChannelID);
-   //
-   // boolean runItNow = false;
-   //
-   // synchronized (replicationLock)
-   // {
-   // if (playedResponsesOnFailure && action != null)
-   // {
-   // // Already replicating channel failed, so just play the action now
-   //
-   // runItNow = true;
-   // }
-   // else
-   // {
-   // if (action != null)
-   // {
-   // responseActions.add(action);
-   //
-   // responseActionCount++;
-   // }
-   //
-   // final MessagingBuffer buffer = connection.getTransportConnection()
-   // .createBuffer(packet.getRequiredBufferSize());
-   //
-   // packet.encode(buffer);
-   //
-   // connection.getTransportConnection().write(buffer);
-   // }
-   // }
-   //
-   // // Execute outside lock
-   //
-   // if (runItNow)
-   // {
-   // action.run();
-   // }
-   // }
-
    public void setCommandConfirmationHandler(final CommandConfirmationHandler handler)
    {
       this.commandConfirmationHandler = handler;
    }
 
-   // public void executeOutstandingDelayedResults()
-   // {
-   // // Execute on different thread to avoid deadlock
-   //
-   // new Thread()
-   // {
-   // public void run()
-   // {
-   // doExecuteOutstandingDelayedResults();
-   // }
-   // }.start();
-   // }
-   //
-   // private void doExecuteOutstandingDelayedResults()
-   // {
-   // List<Runnable> toRun = new ArrayList<Runnable>();
-   //
-   // synchronized (replicationLock)
-   // {
-   // // Execute all the response actions now
-   //
-   // while (true)
-   // {
-   // Runnable action = responseActions.poll();
-   //
-   // if (action != null)
-   // {
-   // toRun.add(action);
-   // }
-   // else
-   // {
-   // break;
-   // }
-   // }
-   //
-   // responseActionCount = 0;
-   //
-   // playedResponsesOnFailure = true;
-   //
-   // for (Runnable action : toRun)
-   // {
-   // action.run();
-   // }
-   // }
-   //
-   // }
-
    public void setHandler(final ChannelHandler handler)
    {
       this.handler = handler;
@@ -521,8 +404,6 @@
 
       for (final Packet packet : resendCache)
       {
-         //log.info("Replaying command " + packet);
-         
          doWrite(packet);
       }
    }
@@ -579,8 +460,7 @@
             receivedBytes = 0;
 
             if (connection.isActive())
-            {
-              // log.info("sending packet confirmed message " + lastConfirmedCommandID);
+            {               
                final Packet confirmed = new PacketsConfirmedMessage(lastConfirmedCommandID);
 
                confirmed.setChannelID(id);
@@ -633,97 +513,8 @@
             handler.handlePacket(packet);
          }
       }
-
-      // replicateComplete();
    }
 
-   // public void waitForAllReplicationResponse()
-   // {
-   // synchronized (replicationLock)
-   // {
-   // long toWait = 10000; // TODO don't hardcode timeout
-   //
-   // long start = System.currentTimeMillis();
-   //
-   // while (responseActionCount > 0 && toWait > 0)
-   // {
-   // try
-   // {
-   // replicationLock.wait();
-   // }
-   // catch (InterruptedException e)
-   // {
-   // }
-   //
-   // long now = System.currentTimeMillis();
-   //
-   // toWait -= now - start;
-   //
-   // start = now;
-   // }
-   //
-   // if (toWait <= 0)
-   // {
-   // log.warn("Timed out waiting for replication responses to return");
-   // }
-   // }
-   // }
-
-   // private void replicateComplete()
-   // {
-   // if (!connection.isActive() && id != 0)
-   // {
-   // // We're on backup and not ping channel so send back a replication response
-   //
-   // Packet packet = new PacketImpl(REPLICATION_RESPONSE);
-   //
-   // packet.setChannelID(2);
-   //
-   // doWrite(packet);
-   // }
-   // }
-
-   // This will never get called concurrently by more than one thread
-
-   // TODO it's not ideal synchronizing this since it forms a contention point with replication
-   // but we need to do this to protect it w.r.t. the check on replicatingChannel
-   // private void replicateResponseReceived()
-   // {
-   // Runnable result = null;
-   //
-   // synchronized (replicationLock)
-   // {
-   // if (playedResponsesOnFailure)
-   // {
-   // return;
-   // }
-   //
-   // result = responseActions.poll();
-   //
-   // if (result == null)
-   // {
-   // throw new IllegalStateException("Cannot find response action");
-   // }
-   // }
-   //
-   // // Must execute outside of lock
-   // if (result != null)
-   // {
-   // result.run();
-   //
-   // // TODO - we can optimise this not to lock every time - only if waiting for all replications to return
-   // synchronized (replicationLock)
-   // {
-   // responseActionCount--;
-   //
-   // if (responseActionCount == 0)
-   // {
-   // replicationLock.notify();
-   // }
-   // }
-   // }
-   // }
-
    private void doWrite(final Packet packet)
    {
       final MessagingBuffer buffer = connection.getTransportConnection().createBuffer(packet.getRequiredBufferSize());
@@ -735,14 +526,15 @@
 
    private void clearUpTo(final int lastConfirmedCommandID)
    {
-      //log.info("client " + connection.isClient() + " clearing up to " + lastConfirmedCommandID);
-      
       final int numberToClear = 1 + lastConfirmedCommandID - firstStoredCommandID;
 
       if (numberToClear < 0)
       {
-         throw new IllegalArgumentException("Invalid lastConfirmedCommandID: " + lastConfirmedCommandID + 
-                                            " firstStoredCommandID " + firstStoredCommandID + " client " + connection.isClient());
+         throw new IllegalArgumentException("Invalid lastConfirmedCommandID: " + lastConfirmedCommandID +
+                                            " firstStoredCommandID " +
+                                            firstStoredCommandID +
+                                            " client " +
+                                            connection.isClient());
       }
 
       int sizeToFree = 0;
@@ -778,76 +570,4 @@
          sendSemaphore.release(sizeToFree);
       }
    }
-
-   private java.util.Queue<QueuedWrite> queuedWrites = new ConcurrentLinkedQueue<QueuedWrite>();
-
-   private final Object replicationLock = new Object();
-
-   // we only include sequence for debug
-   private static class QueuedWrite
-   {
-      Replicator replicator;
-
-      Packet packet;
-
-     // long sequence;
-
-      boolean done;
-   }
- 
-   public void replicationResponseReceived(final Replicator replicator)
-   {
-      synchronized (replicationLock)
-      {         
-         QueuedWrite qw = queuedWrites.peek();
-         
-         //We assume max of only one queued write per channel per replicator for now - this is true for
-         //all actions
-         
-         //And we only send a replication response if we're waiting for a queued write TODO!
-         
-         if (qw.replicator == replicator)
-         {
-            queuedWrites.remove();
-            
-            send(qw.packet);
-            
-            qw = queuedWrites.peek();
-            
-            while (qw != null)
-            {
-               if (qw.done)
-               {
-                  queuedWrites.remove();
-                  
-                  send(qw.packet);
-                  
-                  qw = queuedWrites.peek();
-               }
-               else
-               {
-                  break;
-               }
-            }
-         }
-         else
-         {
-            Iterator<QueuedWrite> iter = queuedWrites.iterator();
-            
-            iter.next();
-            
-            while (iter.hasNext())
-            {
-               qw = iter.next();
-               
-               if (qw.replicator == replicator)
-               {
-                  qw.done = true;
-                  
-                  break;
-               }
-            }
-         }                  
-      }
-   }
 }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/PacketDecoder.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/PacketDecoder.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/PacketDecoder.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -80,6 +80,7 @@
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_XA_SET_TIMEOUT_RESP;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_XA_START;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.SESS_XA_SUSPEND;
+import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.UNREGISTER_QUEUE_REPLICATION_CHANNEL;
 
 import org.jboss.messaging.core.logging.Logger;
 import org.jboss.messaging.core.remoting.Packet;
@@ -134,6 +135,7 @@
 import org.jboss.messaging.core.remoting.impl.wireformat.replication.ReplicateRemoteConsumerRemovedMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.replication.ReplicateStartupInfoMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.replication.ReplicationResponseMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.replication.UnregisterQueueReplicationChannelMessage;
 import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
 
 /**
@@ -445,6 +447,11 @@
             packet = new RegisterQueueReplicationChannelMessage();
             break;
          }
+         case UNREGISTER_QUEUE_REPLICATION_CHANNEL:
+         {
+            packet = new UnregisterQueueReplicationChannelMessage();
+            break;
+         }
          default:
          {
             throw new IllegalArgumentException("Invalid type: " + packetType);

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/RemotingConnectionImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/RemotingConnectionImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/RemotingConnectionImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -27,7 +27,6 @@
 import org.jboss.messaging.core.remoting.Interceptor;
 import org.jboss.messaging.core.remoting.Packet;
 import org.jboss.messaging.core.remoting.RemotingConnection;
-import org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl;
 import org.jboss.messaging.core.remoting.spi.Connection;
 import org.jboss.messaging.core.remoting.spi.ConnectionLifeCycleListener;
 import org.jboss.messaging.core.remoting.spi.Connector;
@@ -85,7 +84,7 @@
    // -----------------------------------------------------------------------------------
 
    private final Connection transportConnection;
-   
+
    private final RemotingConnection replicatingConnection;
 
    private final Map<Long, Channel> channels = new ConcurrentHashMap<Long, Channel>();
@@ -117,9 +116,9 @@
    private boolean frozen;
 
    private final Object failLock = new Object();
-   
+
    private final PacketDecoder decoder = new PacketDecoder();
-   
+
    // Constructors
    // ---------------------------------------------------------------------------------
 
@@ -146,7 +145,7 @@
    }
 
    private RemotingConnectionImpl(final Connection transportConnection,
-                                 final RemotingConnection replicatingConnection,
+                                  final RemotingConnection replicatingConnection,
                                   final long blockingCallTimeout,
                                   final List<Interceptor> interceptors,
                                   final boolean active,
@@ -154,7 +153,7 @@
 
    {
       this.transportConnection = transportConnection;
-      
+
       this.replicatingConnection = replicatingConnection;
 
       this.blockingCallTimeout = blockingCallTimeout;
@@ -336,27 +335,27 @@
    {
       return transferLock;
    }
-   
+
    public boolean isActive()
    {
       return active;
    }
-   
+
    public boolean isClient()
    {
       return client;
    }
-   
+
    public boolean isDestroyed()
    {
       return destroyed;
    }
-   
+
    public long getBlockingCallTimeout()
    {
       return blockingCallTimeout;
    }
-   
+
    public RemotingConnection getReplicatingConnection()
    {
       return replicatingConnection;
@@ -368,7 +367,7 @@
    public void bufferReceived(final Object connectionID, final MessagingBuffer buffer)
    {
       final Packet packet = decoder.decode(buffer);
-      
+
       synchronized (transferLock)
       {
          if (!frozen)
@@ -401,6 +400,10 @@
             {
                channel.handlePacket(packet);
             }
+            else
+            {
+               log.info("cannot find handler for packet " + packet + " client " + this.isClient() + " active " + this.isActive());
+            }
          }
       }
    }
@@ -484,7 +487,7 @@
          channel.close();
       }
    }
-   
+
    private static class DelegatingBufferHandler extends AbstractBufferHandler
    {
       RemotingConnection conn;

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/invm/InVMRegistry.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/invm/InVMRegistry.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/invm/InVMRegistry.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -41,8 +41,7 @@
    private ConcurrentMap<Integer, InVMAcceptor> acceptors = new ConcurrentHashMap<Integer, InVMAcceptor>();
 
    public void registerAcceptor(final int id, final InVMAcceptor acceptor)
-   {     
-      //log.info("Registering acceptor with id " + id, new Exception());
+   {           
       if (acceptors.putIfAbsent(id, acceptor) != null)
       {
          throw new IllegalArgumentException("Acceptor with id " + id + " already registered");
@@ -69,11 +68,6 @@
 
    public int size()
    {
-      log.info("** remaining acceptors");
-      for (Integer i: acceptors.keySet())
-      {
-         log.info("id: " + i);
-      }
       return this.acceptors.size();
    }
 }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/wireformat/PacketImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/wireformat/PacketImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/impl/wireformat/PacketImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -164,8 +164,10 @@
    
    public static final byte REGISTER_QUEUE_REPLICATION_CHANNEL = 100;
    
-   public static final byte REGISTER_POST_OFFICE_REPLICATION_CHANNEL = 101;
+   public static final byte UNREGISTER_QUEUE_REPLICATION_CHANNEL = 101;
    
+   public static final byte REGISTER_POST_OFFICE_REPLICATION_CHANNEL = 102;
+   
    // Static --------------------------------------------------------
 
    public PacketImpl(final byte type)

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/server/impl/RemotingServiceImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/server/impl/RemotingServiceImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/remoting/server/impl/RemotingServiceImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -44,7 +44,6 @@
 import org.jboss.messaging.core.remoting.impl.invm.TransportConstants;
 import org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl;
 import org.jboss.messaging.core.remoting.impl.wireformat.Ping;
-import org.jboss.messaging.core.remoting.impl.wireformat.replication.ReplicationResponseMessage;
 import org.jboss.messaging.core.remoting.server.RemotingService;
 import org.jboss.messaging.core.remoting.spi.Acceptor;
 import org.jboss.messaging.core.remoting.spi.AcceptorFactory;
@@ -235,8 +234,6 @@
       for (Acceptor acceptor : acceptors)
       {
          acceptor.stop();
-         
-         log.info("Stopping acceptor " + acceptor);
       }
 
       acceptors.clear();
@@ -285,10 +282,10 @@
          throw new IllegalStateException("Unable to create connection, server hasn't finished starting up");
       }
 
-      RemotingConnection replicatingConnection = server.getReplicatingConnection();
+      RemotingConnection replicatingConnection = server.getNonPooledReplicatingConnection();
            
       RemotingConnection rc = new RemotingConnectionImpl(connection, replicatingConnection, interceptors, !config.isBackup());
-
+        
       Channel channel1 = rc.getChannel(1, -1, false);
       
       final Replicator replicator;
@@ -304,8 +301,7 @@
             public void handlePacket(final Packet packet)
             {
                if (packet.getType() == PacketImpl.REPLICATION_RESPONSE)
-               {
-                  ReplicationResponseMessage msg = (ReplicationResponseMessage)packet;
+               {                  
                   replicator.replicationResponseReceived();
                }
                else
@@ -434,6 +430,13 @@
       {
          pinger.close();
       }
+      
+      RemotingConnection replConnection = connection.getReplicatingConnection();
+      
+      if (replConnection != null)
+      {
+         server.returnNonPooledReplicatingConnection(replConnection);
+      }
 
       return connection;
    }
@@ -479,7 +482,7 @@
          if (!gotInitialPing)
          {
             // Never received initial ping
-            log.warn("Did not receive initial ping for connection, it will be closed");
+            log.warn(System.identityHashCode(conn) + " Did not receive initial ping for connection, it will be closed");
 
             closeConnection(conn);
 

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/MessagingServer.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/MessagingServer.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/MessagingServer.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -14,6 +14,7 @@
 
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.ExecutorService;
 
 import javax.management.MBeanServer;
 
@@ -22,7 +23,6 @@
 import org.jboss.messaging.core.management.impl.MessagingServerControlImpl;
 import org.jboss.messaging.core.persistence.StorageManager;
 import org.jboss.messaging.core.postoffice.PostOffice;
-import org.jboss.messaging.core.remoting.Channel;
 import org.jboss.messaging.core.remoting.RemotingConnection;
 import org.jboss.messaging.core.remoting.impl.wireformat.CreateSessionResponseMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.ReattachSessionResponseMessage;
@@ -62,15 +62,15 @@
    Version getVersion();
 
    MessagingServerControlImpl getMessagingServerControl();
-   
+
    void registerActivateCallback(ActivateCallback callback);
-   
+
    void unregisterActivateCallback(ActivateCallback callback);
-    
+
    ReattachSessionResponseMessage reattachSession(RemotingConnection connection, String name, int lastConfirmedCommandID) throws Exception;
 
    CreateSessionResponseMessage createSession(String name,
-                                              long channelID,                                              
+                                              long channelID,
                                               String username,
                                               String password,
                                               int minLargeMessageSize,
@@ -95,6 +95,8 @@
 
    HierarchicalRepository<AddressSettings> getAddressSettingsRepository();
 
+   ExecutorService getThreadPool();
+
    int getConnectionCount();
 
    PostOffice getPostOffice();
@@ -109,10 +111,12 @@
 
    SimpleString getNodeID();
 
-   //Channel getReplicatingChannel();
-   
-   RemotingConnection getReplicatingConnection();
+   // Channel getReplicatingChannel();
 
+   RemotingConnection getNonPooledReplicatingConnection();
+
+   void returnNonPooledReplicatingConnection(RemotingConnection connection);
+
    void initialiseBackup(UUID nodeID, long currentMessageID) throws Exception;
 
    boolean isInitialised();
@@ -128,7 +132,7 @@
                      SimpleString filterString,
                      boolean durable,
                      boolean temporary) throws Exception;
-   
+
    void destroyQueue(SimpleString queueName, ServerSession session) throws Exception;
 
    void handleReplicateRedistribution(final SimpleString queueName, final long messageID) throws Exception;

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/Queue.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/Queue.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/Queue.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -27,6 +27,7 @@
 import java.util.concurrent.Executor;
 
 import org.jboss.messaging.core.filter.Filter;
+import org.jboss.messaging.core.server.replication.Replicator;
 import org.jboss.messaging.core.transaction.Transaction;
 import org.jboss.messaging.utils.SimpleString;
 
@@ -44,6 +45,8 @@
    MessageReference reroute(ServerMessage message, Transaction tx) throws Exception;
 
    SimpleString getName();
+   
+   Replicator getReplicator();
 
    long getID();
 
@@ -75,10 +78,12 @@
 
    void cancel(MessageReference reference) throws Exception;
 
-   void deliverAsync(Executor executor);
+   void deliverAsync();
    
-   void deliverAll();
+   void close();
    
+ //  void deliverAll();
+   
    HandleStatus deliverOne();
 
    List<MessageReference> list(Filter filter);

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/QueueFactory.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/QueueFactory.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/QueueFactory.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -22,6 +22,8 @@
 
 package org.jboss.messaging.core.server;
 
+import java.util.concurrent.Executor;
+
 import org.jboss.messaging.core.filter.Filter;
 import org.jboss.messaging.core.postoffice.PostOffice;
 import org.jboss.messaging.core.server.replication.Replicator;
@@ -40,7 +42,7 @@
 public interface QueueFactory
 {
    Queue createQueue(long persistenceID, final SimpleString address, SimpleString name, Filter filter, boolean durable,
-                     boolean temporary, Replicator replicator);
+                     boolean temporary, Replicator replicator, Executor executor);
 
    /**
     * This is required for delete-all-reference to work correctly with paging

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/ServerSession.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/ServerSession.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/ServerSession.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -73,7 +73,7 @@
 
    void removeConsumer(ServerConsumer consumer) throws Exception;
 
-   void close() throws Exception;
+   void closeChannels();
 
    void promptDelivery(Queue queue);
    

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/cluster/impl/BridgeImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/cluster/impl/BridgeImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/cluster/impl/BridgeImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -640,7 +640,7 @@
 
          active = true;
 
-         queue.deliverAsync(executor);
+         queue.deliverAsync();
          
          return true;
       }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/cluster/impl/Redistributor.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/cluster/impl/Redistributor.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/cluster/impl/Redistributor.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -176,7 +176,7 @@
          {
             active = true;
 
-            queue.deliverAsync(executor);
+            queue.deliverAsync();
          }
       }
    }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/LastValueQueue.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/LastValueQueue.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/LastValueQueue.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -27,6 +27,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Executor;
 import java.util.concurrent.ScheduledExecutorService;
 
 import org.jboss.messaging.core.filter.Filter;
@@ -71,7 +72,8 @@
                          final PostOffice postOffice,
                          final StorageManager storageManager,
                          final HierarchicalRepository<AddressSettings> addressSettingsRepository,
-                         final Replicator replicator)
+                         final Replicator replicator,
+                         final Executor executor)
    {
       super(id,
             address,
@@ -83,7 +85,8 @@
             postOffice,
             storageManager,
             addressSettingsRepository,
-            replicator);
+            replicator,
+            executor);
       this.pagingManager = postOffice.getPagingManager();
       this.storageManager = storageManager;
    }

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -72,13 +72,12 @@
 import org.jboss.messaging.core.remoting.ChannelHandler;
 import org.jboss.messaging.core.remoting.Packet;
 import org.jboss.messaging.core.remoting.RemotingConnection;
-import org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl;
 import org.jboss.messaging.core.remoting.impl.wireformat.CreateSessionResponseMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl;
 import org.jboss.messaging.core.remoting.impl.wireformat.ReattachSessionResponseMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.replication.RegisterQueueReplicationChannelMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.replication.ReplicateStartupInfoMessage;
-import org.jboss.messaging.core.remoting.impl.wireformat.replication.ReplicationResponseMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.replication.UnregisterQueueReplicationChannelMessage;
 import org.jboss.messaging.core.remoting.server.RemotingService;
 import org.jboss.messaging.core.remoting.server.impl.RemotingServiceImpl;
 import org.jboss.messaging.core.remoting.spi.Connection;
@@ -286,9 +285,9 @@
       // We start the remoting service here - if the server is a backup remoting service needs to be started
       // so it can be initialised by the live node
       remotingService.start();
-      
+
       started = true;
-      
+
       log.info("JBoss Messaging Server version " + getVersion().getFullVersion() + " started");
    }
 
@@ -311,8 +310,6 @@
          session.getChannel().flushConfirmations();
       }
 
-      log.info("Stopping remoting service on backup " + configuration.isBackup());
-      
       remotingService.stop();
 
       // Stop the deployers
@@ -361,12 +358,17 @@
       {
          postOffice.stop();
       }
-      
-      if (replicatingConnectionManager != null)
+
+      if (nonPooledReplicatingConnectionManager != null)
       {
-         replicatingConnectionManager.close();
+         nonPooledReplicatingConnectionManager.close();
       }
 
+      if (pooledReplicatingConnectionManager != null)
+      {
+         pooledReplicatingConnectionManager.close();
+      }
+
       // Need to shutdown pools before shutting down paging manager to make sure everything is written ok
 
       List<Runnable> tasks = scheduledPool.shutdownNow();
@@ -410,9 +412,7 @@
       sessions.clear();
 
       started = false;
-      
-      log.info(System.identityHashCode(this) + " called stop ");
-      
+
       initialised = false;
       uuid = null;
       nodeID = null;
@@ -486,6 +486,11 @@
    {
       return clusterManager;
    }
+   
+   public ExecutorService getThreadPool()
+   {
+      return this.threadPool;
+   }
 
    public ReattachSessionResponseMessage reattachSession(final RemotingConnection connection,
                                                          final String name,
@@ -495,8 +500,6 @@
 
       // Need to activate the connection even if session can't be found - since otherwise response
       // will never get back
-      
-      log.info("** reattaching session");
 
       checkActivate(connection);
 
@@ -508,9 +511,7 @@
       {
          // Reconnect the channel to the new connection
          int serverLastConfirmedCommandID = session.transferConnection(connection, lastConfirmedCommandID);
-                  
-         log.info("Reattached session ok, server last received command id is " + serverLastConfirmedCommandID);
-         
+
          return new ReattachSessionResponseMessage(serverLastConfirmedCommandID, false);
       }
    }
@@ -732,6 +733,17 @@
       }
 
       postOffice.removeBinding(queueName);
+
+      Replicator replicator = queue.getReplicator();
+
+      if (replicator != null)
+      {
+         Channel channel1 = replicator.getReplicatingChannel().getConnection().getChannel(1, -1, false);
+
+         channel1.send(new UnregisterQueueReplicationChannelMessage(queue.getID()));
+      }
+
+      queue.close();
    }
 
    public synchronized void registerActivateCallback(final ActivateCallback callback)
@@ -748,8 +760,10 @@
 
    private Map<String, Object> backupConnectorParams;
 
-   private ConnectionManager replicatingConnectionManager;
-   
+   private ConnectionManager pooledReplicatingConnectionManager;
+
+   private ConnectionManager nonPooledReplicatingConnectionManager;
+
    private void setupBackupConnectorFactory()
    {
       String backupConnectorName = configuration.getBackupConnectorName();
@@ -779,58 +793,59 @@
             }
 
             backupConnectorParams = backupConnector.getParams();
-            
-            replicatingConnectionManager = new ConnectionManagerImpl(null,
-                                                                     backupConnector,
-                                                                     null,
-                                                                     false,
-                                                                     10, // TODO don't hardcode this
-                                                                     ClientSessionFactoryImpl.DEFAULT_CALL_TIMEOUT,
-                                                                     ClientSessionFactoryImpl.DEFAULT_CLIENT_FAILURE_CHECK_PERIOD,
-                                                                     ClientSessionFactoryImpl.DEFAULT_CONNECTION_TTL,
-                                                                     0,
-                                                                     0d,
-                                                                     0,
-                                                                     this.threadPool,
-                                                                     this.scheduledPool);
+
+            pooledReplicatingConnectionManager = new ConnectionManagerImpl(null, backupConnector, null, false, 10, // TODO
+                                                                           // don't
+                                                                           // hardcode
+                                                                           // this
+                                                                           ClientSessionFactoryImpl.DEFAULT_CALL_TIMEOUT,
+                                                                           ClientSessionFactoryImpl.DEFAULT_CLIENT_FAILURE_CHECK_PERIOD,
+                                                                           ClientSessionFactoryImpl.DEFAULT_CONNECTION_TTL,
+                                                                           0,
+                                                                           0d,
+                                                                           0,
+                                                                           this.threadPool,
+                                                                           this.scheduledPool);
+
+            nonPooledReplicatingConnectionManager = new ConnectionManagerImpl(null,
+                                                                              backupConnector,
+                                                                              null,
+                                                                              false,
+                                                                              Integer.MAX_VALUE,
+                                                                              ClientSessionFactoryImpl.DEFAULT_CALL_TIMEOUT,
+                                                                              ClientSessionFactoryImpl.DEFAULT_CLIENT_FAILURE_CHECK_PERIOD,
+                                                                              ClientSessionFactoryImpl.DEFAULT_CONNECTION_TTL,
+                                                                              0,
+                                                                              0d,
+                                                                              0,
+                                                                              this.threadPool,
+                                                                              this.scheduledPool);
          }
       }
    }
 
    private boolean activatedBackup;
-      
+
    public synchronized RemotingConnection getPooledReplicatingConnection()
    {
       RemotingConnection conn = null;
-      
-      if (replicatingConnectionManager != null)
+
+      if (pooledReplicatingConnectionManager != null)
       {
-         conn = replicatingConnectionManager.getConnection(1);
+         conn = pooledReplicatingConnectionManager.getConnection(1);
       }
-      
+
       return conn;
    }
 
-   public synchronized RemotingConnection getReplicatingConnection()
+   public synchronized RemotingConnection getNonPooledReplicatingConnection()
    {
-      // Note we must always get a new connection each time - since there must
-      // be a one to one correspondence
-      // between connections to clients and replicating connections, since we
-      // need to preserve channel ids
-      // before and after failover
+      RemotingConnection conn = null;
 
-      if (backupConnectorFactory != null)
+      if (nonPooledReplicatingConnectionManager != null)
       {
-         NoCacheConnectionLifeCycleListener listener = new NoCacheConnectionLifeCycleListener();
+         conn = nonPooledReplicatingConnectionManager.getConnection(1);
 
-         RemotingConnectionImpl replicatingConnection = (RemotingConnectionImpl)RemotingConnectionImpl.createConnection(backupConnectorFactory,
-                                                                                                                        backupConnectorParams,
-                                                                                                                        ClientSessionFactoryImpl.DEFAULT_CALL_TIMEOUT,
-                                                                                                                        threadPool,
-                                                                                                                        listener);
-
-         listener.conn = replicatingConnection;
-
          if (!activatedBackup)
          {
             // First time we get channel we send a message down it informing the backup of our node id -
@@ -838,28 +853,28 @@
 
             Packet packet = new ReplicateStartupInfoMessage(uuid, storageManager.getCurrentUniqueID());
 
-            Channel channel1 = replicatingConnection.getChannel(1, -1, false);
+            Channel channel1 = conn.getChannel(1, -1, false);
 
             ChannelHandler prevHandler = channel1.getHandler();
-            
+
             sendOnReplicatingAndWaitForResponse(packet, channel1);
-            
+
             channel1.setHandler(prevHandler);
 
             activatedBackup = true;
          }
 
-         // TODO - pinging + execute outstanding results on channels when failure occurs!!!
-
-         return replicatingConnection;
+         // TODO execute outstanding results when failure occurs
       }
-      else
-      {
-         return null;
-      }
 
+      return conn;
    }
 
+   public synchronized void returnNonPooledReplicatingConnection(final RemotingConnection conn)
+   {
+      nonPooledReplicatingConnectionManager.returnConnection(conn);
+   }
+
    private static class NoCacheConnectionLifeCycleListener implements ConnectionLifeCycleListener
    {
       private RemotingConnection conn;
@@ -936,8 +951,6 @@
             }
 
             configuration.setBackup(false);
-            
-            log.info("set backup to false");
 
             if (clusterManager != null)
             {
@@ -967,29 +980,16 @@
    // can't find message in queue since active was delivered immediately
    private void freezeBackupConnection()
    {
-//      // Sanity check
-//      // All replicated sessions should be on the same connection
-//      RemotingConnection replConnection = null;
+      // // Sanity check
+      // // All replicated sessions should be on the same connection
+      // RemotingConnection replConnection = null;
 
       for (ServerSession session : sessions.values())
       {
          RemotingConnection rc = session.getChannel().getConnection();
 
-//         if (replConnection == null)
-//         {
-//            replConnection = rc;
-//         }
-//         else if (replConnection != rc)
-//         {
-//            throw new IllegalStateException("More than one replicating connection!");
-//         }
          rc.freeze();
       }
-//
-//      if (replConnection != null)
-//      {
-//         replConnection.freeze();
-//      }
    }
 
    private void initialisePart1() throws Exception
@@ -1073,11 +1073,7 @@
                                       configuration.getIDCacheSize(),
                                       configuration.isPersistIDCache());
 
-      clusterQueueStateManager = new ClusterQueueStateManagerImpl(executorFactory,
-                                                                  addressSettingsRepository,
-                                                                  postOffice,
-                                                                  storageManager,
-                                                                  managementService);
+      clusterQueueStateManager = new ClusterQueueStateManagerImpl(this);
 
       messagingServerControl = managementService.registerServer(postOffice,
                                                                 storageManager,
@@ -1160,12 +1156,6 @@
       // Deply any pre-defined diverts
       deployDiverts();
 
-      // // Set-up the replicating connection
-      // if (!setupReplicatingConnection())
-      // {
-      // return;
-      // }
-
       if (configuration.isClustered())
       {
          // This can't be created until node id is set
@@ -1189,8 +1179,6 @@
       pagingManager.startGlobalDepage();
 
       initialised = true;
-
-      log.info(System.identityHashCode(this) + " called initialise part2");
    }
 
    private void deployQueuesFromConfiguration() throws Exception
@@ -1204,94 +1192,10 @@
       }
    }
 
-   // public RemotingConnection getReplicatingConnection()
-   // {
-   //      
-   // }
-
-   // private boolean setupReplicatingConnection() throws Exception
-   // {
-   // String backupConnectorName = configuration.getBackupConnectorName();
-   //
-   // if (backupConnectorName != null)
-   // {
-   // TransportConfiguration backupConnector = configuration.getConnectorConfigurations().get(backupConnectorName);
-   //
-   // if (backupConnector == null)
-   // {
-   // log.warn("connector with name '" + backupConnectorName + "' is not defined in the configuration.");
-   // }
-   // else
-   // {
-   // replicatingConnectionManager = new ConnectionManagerImpl(null,
-   // backupConnector,
-   // null,
-   // false,
-   // 1,
-   // ClientSessionFactoryImpl.DEFAULT_CALL_TIMEOUT,
-   // ClientSessionFactoryImpl.DEFAULT_CLIENT_FAILURE_CHECK_PERIOD,
-   // ClientSessionFactoryImpl.DEFAULT_CONNECTION_TTL,
-   // 0,
-   // 1.0d,
-   // 0,
-   // threadPool,
-   // scheduledPool);
-   //
-   // replicatingConnection = replicatingConnectionManager.getConnection(1);
-   //
-   // if (replicatingConnection != null)
-   // {
-   // replicatingChannel = replicatingConnection.getChannel(2, -1, false);
-   //
-   // replicatingConnection.addFailureListener(new FailureListener()
-   // {
-   // public boolean connectionFailed(MessagingException me)
-   // {
-   // replicatingChannel.executeOutstandingDelayedResults();
-   //
-   // return true;
-   // }
-   // });
-   //
-   // // First time we get channel we send a message down it informing the backup of our node id -
-   // // backup and live must have the same node id
-   //
-   // Packet packet = new ReplicateStartupInfoMessage(uuid, storageManager.getCurrentUniqueID());
-   //
-   // final Future future = new Future();
-   //
-   // replicatingChannel.replicatePacket(packet, 1, new Runnable()
-   // {
-   // public void run()
-   // {
-   // future.run();
-   // }
-   // });
-   //
-   // //This may take a while especially if the journal is large
-   // boolean ok = future.await(60000);
-   //
-   // if (!ok)
-   // {
-   // throw new IllegalStateException("Timed out waiting for response from backup for initialisation");
-   // }
-   // }
-   // else
-   // {
-   // log.warn("Backup server MUST be started before live server. Initialisation will not proceed.");
-   //
-   // return false;
-   // }
-   // }
-   // }
-   //
-   // return true;
-   // }
-   
    private void sendOnReplicatingAndWaitForResponse(final Packet packet, final Channel channel)
    {
       final Future future = new Future();
-      
+
       channel.setHandler(new ChannelHandler()
       {
          public void handlePacket(final Packet packet)
@@ -1299,9 +1203,9 @@
             future.run();
          }
       });
-            
+
       channel.send(packet);
-      
+
       boolean ok = future.await(10000);
 
       if (!ok)
@@ -1309,39 +1213,34 @@
          throw new IllegalStateException("Timed out waiting for response from backup");
       }
    }
-   
+
    private Replicator getReplicatorForQueue(final long queueID)
    {
       RemotingConnection replicatingConnection = this.getPooledReplicatingConnection();
-      
+
       final Replicator replicator;
-      
+
       if (replicatingConnection != null)
       {
          Channel channel1 = replicatingConnection.getChannel(1, -1, false);
-         
+
          JBMThread thread = JBMThread.currentThread();
-         
+
          thread.setNoReplayOrRecord();
-         
-         //sendOnReplicatingAndWaitForResponse(new RegisterQueueReplicationChannelMessage(queueID), channel1);
-         
-         //Actually no need to wait for response
-         
+
          channel1.send(new RegisterQueueReplicationChannelMessage(queueID));
-         
+
          thread.resumeRecording();
-                  
+
          Channel replChannel = replicatingConnection.getChannel(queueID, -1, false);
-         
 
          replicator = new ReplicatorImpl(replChannel);
-         
+
          replChannel.setHandler(new ChannelHandler()
          {
             public void handlePacket(final Packet packet)
-            {               
-               replicator.replicationResponseReceived();      
+            {
+               replicator.replicationResponseReceived();
             }
          });
       }
@@ -1349,7 +1248,7 @@
       {
          replicator = null;
       }
-      
+
       return replicator;
    }
 
@@ -1372,7 +1271,7 @@
          {
             filter = new FilterImpl(queueBindingInfo.getFilterString());
          }
-         
+
          Replicator replicator = getReplicatorForQueue(queueBindingInfo.getPersistenceID());
 
          Queue queue = queueFactory.createQueue(queueBindingInfo.getPersistenceID(),
@@ -1381,7 +1280,8 @@
                                                 filter,
                                                 true,
                                                 false,
-                                                replicator);
+                                                replicator,
+                                                executorFactory.getExecutor());
 
          Binding binding = new LocalQueueBinding(queueBindingInfo.getAddress(), queue, nodeID);
 
@@ -1449,7 +1349,7 @@
                              final boolean durable,
                              final boolean temporary,
                              final boolean ignoreIfExists) throws Exception
-   {      
+   {
       Binding binding = postOffice.getBinding(queueName);
 
       if (binding != null)
@@ -1470,18 +1370,25 @@
       {
          filter = new FilterImpl(filterString);
       }
-      
+
       long queueID;
-      
+
       do
       {
          queueID = storageManager.generateUniqueID();
       }
-      while (queueID == 0 || queueID == 1); //0 and 1 are reserved channels
-      
+      while (queueID == 0 || queueID == 1); // 0 and 1 are reserved channels
+
       Replicator replicator = getReplicatorForQueue(queueID);
 
-      Queue queue = queueFactory.createQueue(queueID, address, queueName, filter, durable, temporary, replicator);
+      Queue queue = queueFactory.createQueue(queueID,
+                                             address,
+                                             queueName,
+                                             filter,
+                                             durable,
+                                             temporary,
+                                             replicator,
+                                             executorFactory.getExecutor());
 
       binding = new LocalQueueBinding(address, queue, nodeID);
 
@@ -1597,20 +1504,22 @@
       {
          // This session may well be on a different connection and different channel id, so we must get rid
          // of it and create another
-         
-         //TODO - is this true any more??
+
+         // TODO - is this true any more??
          currentSession.getChannel().close();
       }
 
       Channel channel = connection.getChannel(channelID, sendWindowSize, false);
-           
+
       RemotingConnection replicatingConnection = connection.getReplicatingConnection();
-      
+
       final Replicator replicator;
 
+      Channel replicatingChannel;
+
       if (replicatingConnection != null)
       {
-         Channel replicatingChannel = replicatingConnection.getChannel(channelID, -1, false);
+         replicatingChannel = replicatingConnection.getChannel(channelID, -1, false);
 
          replicator = new ReplicatorImpl(replicatingChannel);
 
@@ -1619,7 +1528,7 @@
             public void handlePacket(final Packet packet)
             {
                if (packet.getType() == PacketImpl.REPLICATION_RESPONSE)
-               {                                    
+               {
                   replicator.replicationResponseReceived();
                }
                else
@@ -1632,6 +1541,8 @@
       else
       {
          replicator = null;
+
+         replicatingChannel = null;
       }
 
       final ServerSessionImpl session = new ServerSessionImpl(name,
@@ -1648,15 +1559,16 @@
                                                               postOffice,
                                                               resourceManager,
                                                               securityStore,
-                                                              executorFactory.getExecutor(),
+                                                              executorFactory,
                                                               channel,
+                                                              replicatingChannel,
                                                               managementService,
                                                               queueFactory,
                                                               this,
                                                               configuration.getManagementAddress());
 
       sessions.put(name, session);
- 
+
       ServerSessionPacketHandler handler = new ServerSessionPacketHandler(session, replicator, configuration);
 
       session.setHandler(handler);

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/MessagingServerPacketHandler.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/MessagingServerPacketHandler.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/MessagingServerPacketHandler.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -18,6 +18,7 @@
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.REGISTER_QUEUE_REPLICATION_CHANNEL;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.REPLICATE_LOCK_SEQUENCES;
 import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.REPLICATE_STARTUP_INFO;
+import static org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl.UNREGISTER_QUEUE_REPLICATION_CHANNEL;
 
 import java.util.List;
 
@@ -35,6 +36,7 @@
 import org.jboss.messaging.core.remoting.impl.wireformat.replication.ReplicateLockSequenceMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.replication.ReplicateStartupInfoMessage;
 import org.jboss.messaging.core.remoting.impl.wireformat.replication.ReplicationResponseMessage;
+import org.jboss.messaging.core.remoting.impl.wireformat.replication.UnregisterQueueReplicationChannelMessage;
 import org.jboss.messaging.core.server.MessagingServer;
 import org.jboss.messaging.core.server.replication.ReplicableAction;
 import org.jboss.messaging.core.server.replication.Replicator;
@@ -60,8 +62,6 @@
    private Replicator replicator;
 
    private volatile List<Long> sequences;
-   
-  // private volatile boolean requiresReplicationResponse;
 
    public MessagingServerPacketHandler(final MessagingServer server,
                                        final Channel channel1,
@@ -77,7 +77,6 @@
       this.replicator = replicator;
    }
 
-
    public void handlePacket(final Packet packet)
    {
       byte type = packet.getType();
@@ -92,19 +91,15 @@
          case REPLICATE_LOCK_SEQUENCES:
          {
             ReplicateLockSequenceMessage msg = (ReplicateLockSequenceMessage)packet;
-            
-            sequences = msg.getSequences();  
-            
-          //  requiresReplicationResponse = msg.isRequiresResponse();
-            
+
+            sequences = msg.getSequences();
+
             return;
          }
          case REPLICATE_STARTUP_INFO:
          {
             ReplicateStartupInfoMessage msg = (ReplicateStartupInfoMessage)packet;
 
-           // log.info("** got replicate startup info");
-            
             try
             {
                server.initialiseBackup(msg.getNodeID(), msg.getCurrentMessageID());
@@ -117,13 +112,25 @@
             break;
          }
          case REGISTER_QUEUE_REPLICATION_CHANNEL:
-         {           
+         {
             RegisterQueueReplicationChannelMessage msg = (RegisterQueueReplicationChannelMessage)packet;
-                        
+
             Channel channel = connection.getChannel(msg.getBindingID(), -1, false);
-            
+
             channel.setHandler(new QueueReplicationPacketHandler(msg.getBindingID(), server.getPostOffice(), channel));
+
+            break;
+         }
+         case UNREGISTER_QUEUE_REPLICATION_CHANNEL:
+         {            
+            UnregisterQueueReplicationChannelMessage msg = (UnregisterQueueReplicationChannelMessage)packet;
+
+            Channel channel = connection.getChannel(msg.getBindingID(), -1, false);
             
+            channel.setHandler(null);
+
+            channel.close();
+
             break;
          }
          case REGISTER_POST_OFFICE_REPLICATION_CHANNEL:
@@ -140,13 +147,13 @@
                {
                   handleCreateSession(request, sequences == null);
                }
-               
+
                public Packet getPacket()
                {
                   return packet;
                }
             };
-            
+
             if (server.getConfiguration().isBackup())
             {
                JBMThread thread = JBMThread.currentThread();
@@ -154,7 +161,7 @@
                thread.setReplay(sequences);
 
                action.run();
-               
+
                thread.setNoReplayOrRecord();
             }
             else
@@ -185,7 +192,7 @@
          }
       }
       sequences = null;
-      
+
       // send the response message
 
       if (server.getConfiguration().isBackup() || type == REPLICATE_STARTUP_INFO)
@@ -193,7 +200,7 @@
          channel1.send(new ReplicationResponseMessage());
       }
    }
-   
+
    private void handleCreateSession(final CreateSessionMessage request, final boolean activate)
    {
       Packet response;

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueFactoryImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueFactoryImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueFactoryImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -22,6 +22,7 @@
 
 package org.jboss.messaging.core.server.impl;
 
+import java.util.concurrent.Executor;
 import java.util.concurrent.ScheduledExecutorService;
 
 import org.jboss.messaging.core.filter.Filter;
@@ -75,7 +76,8 @@
                             final Filter filter,
                             final boolean durable,
                             final boolean temporary,
-                            final Replicator replicator)
+                            final Replicator replicator,
+                            final Executor executor)
    {
       AddressSettings addressSettings = addressSettingsRepository.getMatch(address.toString());
 
@@ -83,16 +85,17 @@
       if (addressSettings.isLastValueQueue())
       {
          queue = new LastValueQueue(id,
-                                   address,
-                                   name,
-                                   filter,
-                                   durable,
-                                   temporary,
-                                   scheduledExecutor,
-                                   postOffice,
-                                   storageManager,
-                                   addressSettingsRepository,
-                                   replicator);
+                                    address,
+                                    name,
+                                    filter,
+                                    durable,
+                                    temporary,
+                                    scheduledExecutor,
+                                    postOffice,
+                                    storageManager,
+                                    addressSettingsRepository,
+                                    replicator,
+                                    executor);
       }
       else
       {
@@ -106,7 +109,8 @@
                                postOffice,
                                storageManager,
                                addressSettingsRepository,
-                               replicator);
+                               replicator,
+                               executor);
       }
 
       queue.setDistributionPolicy(addressSettings.getDistributionPolicy());

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -144,6 +144,8 @@
    private final Set<Consumer> consumers = new HashSet<Consumer>();
 
    private final Lock lock;
+   
+   private final Executor executor;
 
    public QueueImpl(final long id,
                     final SimpleString address,
@@ -155,7 +157,8 @@
                     final PostOffice postOffice,
                     final StorageManager storageManager,
                     final HierarchicalRepository<AddressSettings> addressSettingsRepository,
-                    final Replicator replicator)
+                    final Replicator replicator,
+                    final Executor executor)
    {
       this.id = id;
 
@@ -190,10 +193,27 @@
 
       scheduledDeliveryHandler = new ScheduledDeliveryHandlerImpl(scheduledExecutor);
 
-      lock = new ReplicationAwareReadWriteLock(name.toString(), 0).writeLock();
+      ReplicationAwareReadWriteLock rwLock = new ReplicationAwareReadWriteLock(name.toString(), 0, true);
+      
+      lock = rwLock.writeLock();
 
       this.replicator = replicator;
+      
+      this.executor = executor;
    }
+   
+   public void close()
+   {
+      if (replicator != null)
+      {
+         replicator.close();
+      }
+   }
+   
+   public Replicator getReplicator()
+   {
+      return replicator;
+   }
 
    // Bindable implementation -------------------------------------------------------------------------------------
 
@@ -376,7 +396,7 @@
       add(ref, true);
    }
 
-   public void deliverAsync(final Executor executor)
+   public void deliverAsync()
    {
       // Prevent too many executors running at once
 
@@ -387,26 +407,11 @@
       }
 
       if (waitingToDeliver.compareAndSet(false, true))
-      {
-         // log.info("delivering async on backup " + backup, new Exception());
+      {         
          executor.execute(deliverRunner);
       }
    }
 
-   // // Only used in testing - do not call directly!
-   // public void deliverNow()
-   // {
-   // lock.lock();
-   // try
-   // {
-   // deliver();
-   // }
-   // finally
-   // {
-   // lock.unlock();
-   // }
-   // }
-
    public void addConsumer(final Consumer consumer) throws Exception
    {
       lock.lock();
@@ -462,7 +467,7 @@
          if (redistributor != null)
          {
             // Just prompt delivery
-            deliverAsync(executor);
+            deliverAsync();
          }
 
          if (delay > 0)
@@ -844,14 +849,14 @@
 
    public int deleteMatchingReferences(final Filter filter) throws Exception
    {
+      int count = 0;
+      
+      Transaction tx = new TransactionImpl(storageManager);
+      
       lock.lock();
 
       try
       {
-         int count = 0;
-
-         Transaction tx = new TransactionImpl(storageManager);
-
          Iterator<MessageReference> iter = messageReferences.iterator();
 
          while (iter.hasNext())
@@ -877,27 +882,27 @@
                count++;
             }
          }
-
-         tx.commit();
-
-         return count;
       }
       finally
       {
          lock.unlock();
       }
+      
+      tx.commit();
+
+      return count;
    }
 
    public boolean deleteReference(final long messageID) throws Exception
    {
+      boolean deleted = false;
+
+      Transaction tx = new TransactionImpl(storageManager);
+
       lock.lock();
 
       try
-      {
-         boolean deleted = false;
-
-         Transaction tx = new TransactionImpl(storageManager);
-
+      {         
          Iterator<MessageReference> iter = messageReferences.iterator();
 
          while (iter.hasNext())
@@ -911,16 +916,16 @@
                deleted = true;
                break;
             }
-         }
-
-         tx.commit();
-
-         return deleted;
+         }         
       }
       finally
       {
          lock.unlock();
       }
+      
+      tx.commit();
+      
+      return deleted;
    }
 
    public boolean expireReference(final long messageID) throws Exception
@@ -952,13 +957,14 @@
 
    public int expireReferences(final Filter filter) throws Exception
    {
+      Transaction tx = new TransactionImpl(storageManager);
+
+      int count = 0;
+      
       lock.lock();
 
       try
-      {
-         Transaction tx = new TransactionImpl(storageManager);
-
-         int count = 0;
+      {         
          Iterator<MessageReference> iter = messageReferences.iterator();
 
          while (iter.hasNext())
@@ -971,16 +977,16 @@
                iter.remove();
                count++;
             }
-         }
-
-         tx.commit();
-
-         return count;
+         }        
       }
       finally
       {
          lock.unlock();
       }
+      
+      tx.commit();
+
+      return count;
    }
 
    public void expireReferences() throws Exception
@@ -1059,13 +1065,14 @@
 
    public int moveReferences(final Filter filter, final SimpleString toAddress) throws Exception
    {
+      Transaction tx = new TransactionImpl(storageManager);
+
+      int count = 0;
+      
       lock.lock();
 
       try
-      {
-         Transaction tx = new TransactionImpl(storageManager);
-
-         int count = 0;
+      {         
          Iterator<MessageReference> iter = messageReferences.iterator();
 
          while (iter.hasNext())
@@ -1090,16 +1097,16 @@
                acknowledge(tx, ref);
                count++;
             }
-         }
-
-         tx.commit();
-
-         return count;
+         }        
       }
       finally
       {
          lock.unlock();
       }
+      
+      tx.commit();
+
+      return count;
    }
 
    public boolean changeReferencePriority(final long messageID, final byte newPriority) throws Exception
@@ -1171,7 +1178,7 @@
 
          scheduledDeliveryHandler.reSchedule();
 
-         deliverAsync(executor);
+         deliverAsync();
       }
    }
 
@@ -1247,7 +1254,7 @@
 
          redistributor.start();
 
-         deliverAsync(executor);
+         deliverAsync();
       }
    }
 
@@ -1529,8 +1536,7 @@
          if (direct)
          {
             // Deliver directly
-
-            // log.info("delivering direct on backup " + backup);
+            
             HandleStatus status = deliverReference(ref);
 
             if (status == HandleStatus.HANDLED)
@@ -1552,8 +1558,7 @@
             }
          }
          else
-         {
-            // log.info("Not delivering direct on backup " + backup);
+         {            
             add = true;
          }
 
@@ -1573,16 +1578,16 @@
                messageReferences.addLast(ref, ref.getMessage().getPriority());
             }
 
-            if (!direct && promptDelivery)
-            {
-               // We have consumers with filters which don't match, so we need
-               // to prompt delivery every time
-               // a new message arrives - this is why you really shouldn't use
-               // filters with queues - in most cases
-               // it's an ant-pattern since it would cause a queue scan on each
-               // message
-               deliverAll();
-            }
+//            if (!direct && promptDelivery)
+//            {
+//               // We have consumers with filters which don't match, so we need
+//               // to prompt delivery every time
+//               // a new message arrives - this is why you really shouldn't use
+//               // filters with queues - in most cases
+//               // it's an ant-pattern since it would cause a queue scan on each
+//               // message
+//               deliverAll();
+//            }
          }
       }
       finally
@@ -1680,9 +1685,9 @@
          lock.unlock();
       }
 
-      deliverAll();
+      deliverAsync();
    }
-
+   
    // Inner classes
    // --------------------------------------------------------------------------
 
@@ -1693,8 +1698,6 @@
          // Must be set to false *before* executing to avoid race
          waitingToDeliver.set(false);
 
-         // log.info("** calling deliver runner " + backup, new Exception());
-
          deliverAll();
       }
    }
@@ -1702,7 +1705,7 @@
    /*
     * Attempt to deliver all the messages in the queue
     */
-   public void deliverAll()
+   private void deliverAll()
    {
       // direct = false;
 

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueReplicationPacketHandler.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueReplicationPacketHandler.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/QueueReplicationPacketHandler.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -56,8 +56,6 @@
 
    private volatile List<Long> sequences;
    
-   //private volatile boolean requiresReplicationResponse;
-   
    private final PostOffice postOffice;
    
    private final Channel channel;
@@ -80,9 +78,7 @@
             ReplicateLockSequenceMessage msg = (ReplicateLockSequenceMessage)packet;
             
             sequences = msg.getSequences();
-            
-            //requiresReplicationResponse = msg.isRequiresResponse();
-            
+ 
             break;
          }
          case PacketImpl.REPLICATE_QUEUE_DELIVERY:
@@ -107,13 +103,8 @@
                JBMThread.currentThread().resumeReplay();
             }
             
-            HandleStatus handled = queue.deliverOne();
+            queue.deliverOne();
             
-//            if (handled == HandleStatus.HANDLED)
-//            {
-//               log.info("*** delivered message on backup");
-//            }
-            
             channel.send(new ReplicationResponseMessage());            
             
             thread.setNoReplayOrRecord();

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -135,7 +135,7 @@
                              final Channel channel,
                              final boolean preAcknowledge,
                              final boolean updateDeliveries,
-                             final Executor executor,
+                            // final Executor executor,
                              final ManagementService managementService) throws Exception
    {
       this.id = id;
@@ -148,7 +148,7 @@
 
       this.messageQueue = binding.getQueue();
 
-      this.executor = executor;
+      this.executor = null;
 
       this.started = browseOnly || started;
 
@@ -168,8 +168,10 @@
 
       this.updateDeliveries = updateDeliveries;
 
-      lock = new ReplicationAwareReadWriteLock("consumer " + id, 0).writeLock();
-
+      ReplicationAwareReadWriteLock rwLock = new ReplicationAwareReadWriteLock("consumer " + id, 0, true);
+      
+      lock = rwLock.writeLock();
+            
       binding.getQueue().addConsumer(this);
    }
 
@@ -487,11 +489,8 @@
 
       try
       {
-         //log.info("handling message");
-         
          if ((flowControl && availableCredits <= 0) || !started)
-         {
-            //log.info("busy");
+         {            
             return HandleStatus.BUSY;
          }
 

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -19,7 +19,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executor;
 
 import javax.transaction.xa.XAException;
 import javax.transaction.xa.XAResource;
@@ -91,6 +90,7 @@
 import org.jboss.messaging.core.transaction.ResourceManager;
 import org.jboss.messaging.core.transaction.Transaction;
 import org.jboss.messaging.core.transaction.impl.TransactionImpl;
+import org.jboss.messaging.utils.ExecutorFactory;
 import org.jboss.messaging.utils.IDGenerator;
 import org.jboss.messaging.utils.SimpleIDGenerator;
 import org.jboss.messaging.utils.SimpleString;
@@ -134,7 +134,7 @@
 
    private final Map<Long, ServerConsumer> consumers = new ConcurrentHashMap<Long, ServerConsumer>();
 
-   private final Executor executor;
+   private final ExecutorFactory executorFactory;
 
    private Transaction tx;
 
@@ -148,6 +148,8 @@
 
    private final Channel channel;
 
+   private final Channel replicatingChannel;
+
    private final ManagementService managementService;
 
    private volatile boolean started = false;
@@ -175,7 +177,6 @@
    // Constructors ---------------------------------------------------------------------------------
 
    public ServerSessionImpl(final String name,
-   // final long oppositeChannelID,
                             final String username,
                             final String password,
                             final int minLargeMessageSize,
@@ -189,20 +190,16 @@
                             final PostOffice postOffice,
                             final ResourceManager resourceManager,
                             final SecurityStore securityStore,
-                            final Executor executor,
+                            final ExecutorFactory executorFactory,
                             final Channel channel,
+                            final Channel replicatingChannel,
                             final ManagementService managementService,
                             final QueueFactory queueFactory,
                             final MessagingServer server,
-                            final SimpleString managementAddress
-   // final Channel replicatingChannel,
-   // final boolean backup
-   ) throws Exception
+                            final SimpleString managementAddress) throws Exception
    {
       this.id = channel.getID();
 
-      // this.oppositeChannelID = oppositeChannelID;
-
       this.username = username;
 
       this.password = password;
@@ -225,7 +222,7 @@
 
       this.securityStore = securityStore;
 
-      this.executor = executor;
+      this.executorFactory = executorFactory;
 
       if (!xa)
       {
@@ -236,6 +233,8 @@
 
       this.channel = channel;
 
+      this.replicatingChannel = replicatingChannel;
+
       this.managementService = managementService;
 
       this.name = name;
@@ -248,10 +247,6 @@
 
       this.nodeID = server.getNodeID();
 
-      // this.replicatingChannel = replicatingChannel;
-
-      // this.backup = backup;
-
       remotingConnection.addFailureListener(this);
 
       remotingConnection.addCloseListener(this);
@@ -307,51 +302,14 @@
       }
    }
 
-   public void close() throws Exception
-   {
-      if (tx != null && tx.getXid() == null)
-      {
-         // We only rollback local txs on close, not XA tx branches
-
-         rollback(false);
-      }
-
-      Set<ServerConsumer> consumersClone = new HashSet<ServerConsumer>(consumers.values());
-
-      for (ServerConsumer consumer : consumersClone)
-      {
-         consumer.close();
-      }
-
-      consumers.clear();
-
-      server.removeSession(name);
-
-      if (currentLargeMessage != null)
-      {
-         try
-         {
-            currentLargeMessage.deleteFile();
-         }
-         catch (Throwable error)
-         {
-            log.error("Failed to delete large message file", error);
-         }
-      }
-
-      remotingConnection.removeFailureListener(this);
-   }
-
    public void promptDelivery(final Queue queue)
    {
-      queue.deliverAsync(executor);
+      queue.deliverAsync();
    }
 
    public void handleStart(final Packet packet)
    {
       setStarted(true);
-
-      ////channel.confirm(packet);
    }
 
    public void handleStop(final Packet packet)
@@ -360,7 +318,7 @@
 
       setStarted(false);
 
-      ////channel.confirm(packet);
+      // //channel.confirm(packet);
 
       channel.send(response);
    }
@@ -430,7 +388,7 @@
          }
       }
 
-      ////channel.confirm(packet);
+      // //channel.confirm(packet);
 
       channel.send(response);
    }
@@ -476,7 +434,8 @@
                                                 filter,
                                                 false,
                                                 true,
-                                                null);
+                                                null,
+                                                executorFactory.getExecutor());
 
             // There's no need for any special locking since the list method is synchronized
             List<MessageReference> refs = ((Queue)binding.getBindable()).list(filter);
@@ -504,7 +463,6 @@
                                                           channel,
                                                           preAcknowledge,
                                                           updateDeliveries,
-                                                          executor,
                                                           managementService);
 
          consumers.put(consumer.getID(), consumer);
@@ -549,13 +507,13 @@
          }
       }
 
-      ////channel.confirm(packet);
+      // //channel.confirm(packet);
 
       channel.send(response);
    }
 
    public void handleCreateQueue(final CreateQueueMessage packet)
-   {      
+   {
       SimpleString address = packet.getAddress();
 
       final SimpleString name = packet.getQueueName();
@@ -625,7 +583,7 @@
          }
       }
 
-      ////channel.confirm(packet);
+      // //channel.confirm(packet);
 
       channel.send(response);
    }
@@ -663,7 +621,7 @@
          }
       }
 
-      ////channel.confirm(packet);
+      // //channel.confirm(packet);
 
       channel.send(response);
    }
@@ -716,7 +674,7 @@
          }
       }
 
-      ////channel.confirm(packet);
+      // //channel.confirm(packet);
 
       channel.send(response);
    }
@@ -762,7 +720,7 @@
          }
       }
 
-      ////channel.confirm(packet);
+      // //channel.confirm(packet);
 
       channel.send(response);
    }
@@ -799,7 +757,7 @@
          }
       }
 
-      ////channel.confirm(packet);
+      // //channel.confirm(packet);
 
       if (response != null)
       {
@@ -824,7 +782,7 @@
          log.error("Failed to acknowledge", e);
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
    }
 
    public void handleCommit(final Packet packet)
@@ -855,7 +813,7 @@
          tx = new TransactionImpl(storageManager);
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -884,7 +842,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -947,7 +905,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1021,7 +979,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1033,7 +991,7 @@
 
       Packet response = new SessionXAResponseMessage(false, XAResource.XA_OK, null);
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1084,7 +1042,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1146,7 +1104,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1209,7 +1167,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1260,7 +1218,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1309,7 +1267,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1369,7 +1327,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1378,7 +1336,7 @@
    {
       Packet response = new SessionXAGetInDoubtXidsResponseMessage(resourceManager.getPreparedTransactions());
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1387,7 +1345,7 @@
    {
       Packet response = new SessionXAGetTimeoutResponseMessage(resourceManager.getTimeoutSeconds());
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1396,7 +1354,7 @@
    {
       Packet response = new SessionXASetTimeoutResponseMessage(resourceManager.setTimeoutSeconds(packet.getTimeoutSeconds()));
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       channel.send(response);
    }
@@ -1446,7 +1404,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       if (response != null)
       {
@@ -1503,7 +1461,7 @@
          }
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
 
       if (response != null)
       {
@@ -1517,8 +1475,38 @@
 
       try
       {
-         close();
+         if (tx != null && tx.getXid() == null)
+         {
+            // We only rollback local txs on close, not XA tx branches
 
+            rollback(false);
+         }
+
+         Set<ServerConsumer> consumersClone = new HashSet<ServerConsumer>(consumers.values());
+
+         for (ServerConsumer consumer : consumersClone)
+         {
+            consumer.close();
+         }
+
+         consumers.clear();
+
+         server.removeSession(name);
+
+         if (currentLargeMessage != null)
+         {
+            try
+            {
+               currentLargeMessage.deleteFile();
+            }
+            catch (Throwable error)
+            {
+               log.error("Failed to delete large message file", error);
+            }
+         }
+
+         remotingConnection.removeFailureListener(this);
+
          response = new NullResponseMessage();
       }
       catch (Exception e)
@@ -1535,14 +1523,21 @@
          }
       }
 
-      //channel.confirm(packet);
-
-      // We flush the confirmations to make sure any send confirmations get handled on the client side
+      channel.confirm(packet);
+      
       channel.flushConfirmations();
+      
+      channel.send(response); 
+   }
 
-      channel.send(response);
-
+   public void closeChannels()
+   {
       channel.close();
+
+      if (replicatingChannel != null)
+      {
+         replicatingChannel.close();
+      }
    }
 
    public int transferConnection(final RemotingConnection newConnection, final int lastConfirmedCommandID)
@@ -1553,11 +1548,7 @@
       {
          this.setStarted(false);
       }
-      
-      log.info("Transferring connection");
 
-      // backup = false;
-
       remotingConnection.removeFailureListener(this);
       remotingConnection.removeCloseListener(this);
 
@@ -1579,7 +1570,6 @@
 
       int serverLastConfirmedCommandID = channel.getLastConfirmedCommandID();
 
-      log.info("replaying commands");
       channel.replayCommands(lastConfirmedCommandID);
 
       if (wasStarted)
@@ -1587,8 +1577,6 @@
          this.setStarted(true);
       }
 
-      log.info("Transferred connection");
-      
       return serverLastConfirmedCommandID;
    }
 
@@ -1698,7 +1686,7 @@
       {
          Packet response = null;
 
-         //channel.confirm(packet);
+         // channel.confirm(packet);
 
          if (response != null)
          {
@@ -1712,16 +1700,16 @@
    private void doReceiveCredits(final SessionConsumerFlowCreditMessage packet)
    {
       try
-      { 
+      {
          ServerConsumer consumer = consumers.get(packet.getConsumerID());
-         
+
          consumer.receiveCredits(packet.getCredits());
       }
       catch (Exception e)
       {
          log.error("Failed to receive credits", e);
       }
-      //channel.confirm(packet);
+      // channel.confirm(packet);
    }
 
    private void doSendLargeMessage(final SessionSendLargeMessage packet)
@@ -1737,7 +1725,7 @@
          log.error("Failed to send message", e);
       }
 
-      //channel.confirm(packet);
+      // channel.confirm(packet);
    }
 
    private void handleManagementMessage(final ServerMessage message) throws Exception

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerSessionPacketHandler.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerSessionPacketHandler.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/impl/ServerSessionPacketHandler.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -104,8 +104,6 @@
    
    private volatile List<Long> sequences;
    
- //  private volatile boolean requiresReplicationResponse;
-
    private final Channel channel;
 
    public ServerSessionPacketHandler(final ServerSession session,
@@ -140,8 +138,6 @@
    {
       this.packet = packet;
       
-     // log.info("Got packet " + packet + " at server session packet handler backup is " + config.isBackup());
-
       if (config.isBackup())
       {
          JBMThread thread = JBMThread.currentThread();
@@ -154,13 +150,14 @@
                   
          if (packet.getType() != PacketImpl.REPLICATE_LOCK_SEQUENCES)
          {
-            channel.confirm(packet);
-            
-            //log.info("sending back replication response");
+            checkConfirm(packet);
+                       
             channel.send(new ReplicationResponseMessage());
          }
          
          thread.setNoReplayOrRecord();
+         
+         checkCloseSessionChannels(packet);         
       }
       else
       {
@@ -171,7 +168,9 @@
             {
                public void run()
                {
-                  channel.confirm(packet);
+                  checkConfirm(packet);
+                  
+                  checkCloseSessionChannels(packet);
                }
             });
          }
@@ -179,19 +178,40 @@
          {
             handlePacket();
             
-            channel.confirm(packet);
+            checkConfirm(packet);
+            
+            checkCloseSessionChannels(packet);
          }
       }
    }
-
-   private void dumpSequences(List<Long> sequences)
+   
+   private void checkConfirm(final Packet packet)
    {
-      for (long sequence : sequences)
+      //TODO this is a bit hacky
+      if (packet.getType() != PacketImpl.SESS_CLOSE)
       {
-         log.info(sequence);
+         channel.confirm(packet);
       }
    }
+   
+   private void checkCloseSessionChannels(final Packet packet)
+   {
+      if (packet.getType() == PacketImpl.SESS_CLOSE)
+      {
+         //Close channels once we have the response back from the backup
+         session.closeChannels();
+      }
+   }
 
+//   private static synchronized void dumpSequences(List<Long> sequences)
+//   {
+//      log.info("dumping sequences");
+//      for (long sequence : sequences)
+//      {
+//         log.info(sequence);
+//      }
+//   }
+
    private void handlePacket()
    {
       byte type = packet.getType();
@@ -206,10 +226,8 @@
 
                sequences = msg.getSequences();
                
-             //  this.requiresReplicationResponse = msg.isRequiresResponse();
-
-               // dumpSequences(sequences);
-
+               //dumpSequences(sequences);
+               
                break;
             }
             case SESS_CREATECONSUMER:

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/Replicator.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/Replicator.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/Replicator.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -24,6 +24,7 @@
 package org.jboss.messaging.core.server.replication;
 
 import org.jboss.messaging.core.remoting.Channel;
+import org.jboss.messaging.core.remoting.impl.QueuedWriteManager;
 
 
 /**
@@ -37,7 +38,11 @@
 {
    void execute(ReplicableAction action, Runnable postReplicateAction);
    
-   void registerWaitingChannel(Channel channel);
+   void registerWaitingChannel(QueuedWriteManager manager);
    
    void replicationResponseReceived();
+   
+   void close();
+   
+   Channel getReplicatingChannel();
 }

Deleted: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/PriorityLock.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/PriorityLock.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/PriorityLock.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -1,131 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source
- * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
- * by the @authors tag. See the copyright.txt in the distribution for a
- * full listing of individual contributors.
- *
- * This is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License as
- * published by the Free Software Foundation; either version 2.1 of
- * the License, or (at your option) any later version.
- *
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this software; if not, write to the Free
- * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
- */
-
-package org.jboss.messaging.core.server.replication.impl;
-
-import java.util.PriorityQueue;
-import java.util.Queue;
-import java.util.concurrent.locks.LockSupport;
-
-import org.jboss.messaging.core.logging.Logger;
-
-/**
- * A PriorityLock
- *
- * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
- *
- *
- */
-public class PriorityLock
-{
-   private static final Logger log = Logger.getLogger(PriorityLock.class);
-
-   private final Queue<QueueEntry> waiting;
-
-   private volatile long currentSequence;
-
-   private Thread owner;
-
-   public PriorityLock(final long sequence)
-   {
-      waiting = new PriorityQueue<QueueEntry>();
-
-      this.currentSequence = sequence;
-   }
-
-   public void lock(final long sequence)
-   {
-      Thread currentThread = Thread.currentThread();
-
-      if (sequence != currentSequence)
-      {
-         QueueEntry entry = new QueueEntry(sequence, currentThread);
-
-         synchronized (waiting)
-         {
-            waiting.add(entry);
-         }
-
-         while (sequence != currentSequence)
-         {
-            LockSupport.park();
-         }
-      }
-
-      owner = currentThread;
-   }
-
-   public void unlock()
-   {
-      if (owner != Thread.currentThread())
-      {
-         throw new IllegalMonitorStateException();
-      }
-
-      currentSequence++;
-
-      QueueEntry entry;
-
-      synchronized (waiting)
-      {
-         entry = waiting.peek();
-
-         if (entry != null && entry.thread == owner)
-         {
-            waiting.poll();
-         }
-
-         entry = waiting.peek();
-      }
-
-      if (entry != null)
-      {
-         LockSupport.unpark(entry.thread);
-      }
-   }
-
-   private static final class QueueEntry implements Comparable<QueueEntry>
-   {
-      private final long sequence;
-
-      private final Thread thread;
-
-      private QueueEntry(final long sequence, final Thread thread)
-      {
-         this.sequence = sequence;
-
-         if (thread == null)
-         {
-            throw new NullPointerException("Null thread");
-         }
-
-         this.thread = thread;
-      }
-
-      public int compareTo(final QueueEntry entry)
-      {
-         long l = entry.sequence;
-
-         return sequence < l ? -1 : (sequence == l ? 0 : 1);
-      }
-   }
-}

Deleted: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareAtomicLong.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareAtomicLong.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareAtomicLong.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -1,96 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source
- * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
- * by the @authors tag. See the copyright.txt in the distribution for a
- * full listing of individual contributors.
- *
- * This is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License as
- * published by the Free Software Foundation; either version 2.1 of
- * the License, or (at your option) any later version.
- *
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this software; if not, write to the Free
- * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
- */
-
-package org.jboss.messaging.core.server.replication.impl;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.jboss.messaging.core.logging.Logger;
-import org.jboss.messaging.utils.Pair;
-
-/**
- * A ReplicationAwareAtomicLong
- *
- * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
- *
- *
- */
-public class ReplicationAwareAtomicLong
-{
-   private static final Logger log = Logger.getLogger(ReplicationAwareAtomicLong.class);
-
-   private final AtomicLong al;
-
-   public ReplicationAwareAtomicLong(final long initialValue)
-   {
-      this.al = new AtomicLong(initialValue);
-   }
-
-   public void set(final long l)
-   {
-      al.set(l);
-   }
-
-   public long getAndIncrement()
-   {
-      JBMThread thread = JBMThread.currentThread();
-
-      if (thread.isReplay())
-      {
-         long sequence = thread.getNextSequence();
-
-         while (!al.compareAndSet(sequence, sequence + 1))
-         {
-            Thread.yield();
-         }
-
-         return sequence;
-      }
-      else
-      {
-         long sequence = al.getAndIncrement();
-
-         if (thread.isRecording())
-         {
-            thread.addSequence(sequence);
-         }
-         
-         return sequence;
-      }
-   }
-   
-   public long get()
-   {
-      return al.get();
-   }
-
-   // public long getAndIncrement(long expected)
-   // {
-   // while (!al.compareAndSet(expected, expected + 1))
-   // {
-   // Thread.yield();
-   // }
-   //
-   // return expected;
-   // }
-
-}

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareReadWriteLock.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareReadWriteLock.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareReadWriteLock.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -22,6 +22,9 @@
 
 package org.jboss.messaging.core.server.replication.impl;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -46,64 +49,107 @@
 {
    private static final Logger log = Logger.getLogger(ReplicationAwareReadWriteLock.class);
 
-   private final Lock readLock = new StatefulObjectReadLock();
-
    private final Lock writeLock = new StatefulObjectWriteLock();
 
- //  private long objectID;
-
    private final ReadWriteLock rwLock;
 
    private final AtomicInteger counter;
 
-   private final PriorityLock sequencedLock;
+   private final SequencedLock sequencedLock;
 
    private String name;
 
-   public ReplicationAwareReadWriteLock(final String name, final int initialCount)
+   private volatile boolean debug;
+
+  // private Map<Long, StackTraceElement[]> acquirers;
+
+   public ReplicationAwareReadWriteLock(final String name, final int initialCount, final boolean debug)
    {
       this.name = name;
 
-    //  this.objectID = objectID;
-
       rwLock = new ReentrantReadWriteLock();
 
-      sequencedLock = new PriorityLock(initialCount);
+      sequencedLock = new SequencedLock(initialCount);
 
       counter = new AtomicInteger(initialCount);
+      
+      this.debug = debug;
+      
+//      if (debug)
+//      {
+//         acquirers = new HashMap<Long, StackTraceElement[]>();
+//         
+//         addLock(this);
+//      }
    }
 
    public Lock readLock()
    {
-      return readLock;
+      return writeLock;      
    }
 
    public Lock writeLock()
    {
       return writeLock;
    }
-   
-   private Map<JBMThread, StackTraceElement[]> stackTraces = new ConcurrentHashMap<JBMThread, StackTraceElement[]>();
-   
-   //debug only
+
+ //  private Map<JBMThread, StackTraceElement[]> stackTraces = new ConcurrentHashMap<JBMThread, StackTraceElement[]>();
+
+   // debug only
    private void addOwner(final JBMThread thread)
    {
       owners.add(thread);
-      
-      stackTraces.put(thread, thread.getStackTrace());
+
+      //stackTraces.put(thread, thread.getStackTrace());
    }
-   
-   //debug only
+
+   // debug only
    private void removeOwner(final JBMThread thread)
    {
       owners.remove(thread);
-      
-      stackTraces.remove(thread);
+
+     // stackTraces.remove(thread);
    }
 
    // For debug
    private Set<Thread> owners = new ConcurrentHashSet<Thread>();
 
+  // private static Map<String, List<ReplicationAwareReadWriteLock>> staticMap = new HashMap<String, List<ReplicationAwareReadWriteLock>>(); 
+   
+//   private static synchronized void addLock(final ReplicationAwareReadWriteLock lock)
+//   {
+//      List<ReplicationAwareReadWriteLock> locks = staticMap.get(lock.name);
+//      
+//      if (locks == null)
+//      {
+//         locks = new ArrayList<ReplicationAwareReadWriteLock>();
+//         
+//         staticMap.put(lock.name, locks);
+//      }
+//      
+//      locks.add(lock);
+//   }
+   
+//   private synchronized static void dumpLocksWithName(final String lockName)
+//   {
+//      List<ReplicationAwareReadWriteLock> locks = staticMap.get(lockName);
+//      
+//      log.info("******* DUMPING LOCKS WITH NAME " + lockName);
+//      log.info("There are " + locks.size() + " locks with name " + lockName);
+//      
+//      int count = 0;
+//      for (ReplicationAwareReadWriteLock lock: locks)
+//      {
+//         log.info("**** DUMPING LOCK " + count++);
+//         for (Map.Entry<Long, StackTraceElement[]> entry : lock.acquirers.entrySet())
+//         {
+//            Exception e = new Exception();
+//            e.setStackTrace(entry.getValue());
+//            log.info("sequence: " + entry.getKey(), e);
+//         }
+//      }
+//   }
+
    private boolean doLock(long time, TimeUnit unit, boolean read) throws InterruptedException
    {
       JBMThread thread = JBMThread.currentThread();
@@ -111,20 +157,28 @@
       // debug only
       if (owners.contains(thread))
       {
-         Exception e = new Exception();
-         e.setStackTrace(stackTraces.get(thread));
-         log.error("Stateful lock is not re-entrant, first obtained here", e);
-         Exception e2 = new Exception();
-         log.info("Second attempt to obtain here", e2);
-         throw new IllegalStateException("Stateful lock is NOT re-entrant!");
+//         Exception e = new Exception();
+//         e.setStackTrace(stackTraces.get(thread));
+//         log.error("Stateful staticLock is not re-entrant, first obtained here", e);
+//         Exception e2 = new Exception();
+//         log.info("Second attempt to obtain here", e2);
+         throw new IllegalStateException("Lock is NOT re-entrant!");
       }
 
       if (thread.isReplay())
       {
          long sequence = thread.getNextSequence();
 
-         sequencedLock.lock(sequence);
+//         if (debug)
+//         {
+//            this.acquirers.put(sequence, thread.getStackTrace());
+//         }
 
+         if (!sequencedLock.lock(sequence))
+         {
+           // dumpLocksWithName(name);
+         }
+
          addOwner(thread);
 
          return true;
@@ -146,7 +200,14 @@
          {
             if (thread.isRecording())
             {
-               thread.addSequence(counter.getAndIncrement());
+               long sequence = counter.getAndIncrement();
+
+               thread.addSequence(sequence);
+
+//               if (debug)
+//               {
+//                  this.acquirers.put(sequence, thread.getStackTrace());
+//               }
             }
 
             addOwner(thread);

Copied: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareSharedCounter.java (from rev 7515, branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareAtomicLong.java)
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareSharedCounter.java	                        (rev 0)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicationAwareSharedCounter.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -0,0 +1,127 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.core.server.replication.impl;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.jboss.messaging.core.logging.Logger;
+import org.jboss.messaging.utils.Pair;
+
+/**
+ * A ReplicationAwareSharedCounter
+ *
+ * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
+ *
+ *
+ */
+public class ReplicationAwareSharedCounter
+{
+   private static final Logger log = Logger.getLogger(ReplicationAwareSharedCounter.class);
+
+   private final AtomicLong al;
+   
+   private final boolean backup;
+
+   public ReplicationAwareSharedCounter(final boolean backup, final long initialValue)
+   {
+      this.al = new AtomicLong(initialValue);
+      
+      this.backup = backup;
+   }
+
+   public void set(final long l)
+   {
+      al.set(l);
+   }
+
+   public long getAndIncrement()
+   {
+      JBMThread thread = JBMThread.currentThread();
+
+//      if (thread.isReplay())
+//      {
+      if (backup)
+      {
+         if (!thread.isReplay())
+         {
+            throw new IllegalStateException("Thread should be in replay mode");
+         }
+         
+         if (thread.isRecording())
+         {
+            throw new IllegalStateException("Thread should not be recording");
+         }
+         
+         long sequence = thread.getNextSequence();
+         
+         synchronized (this)
+         {
+            if (sequence >= al.get())
+            {
+               al.set(sequence + 1);
+            }
+         }
+            
+   
+//            log.info(System.identityHashCode(this) + " attempting to get sequence " + sequence);
+//   
+//            while (!al.compareAndSet(sequence, sequence + 1))
+//            {
+//               Thread.yield();
+//            }
+//   
+//            log.info(System.identityHashCode(this) + " got sequence " + sequence);
+   
+         return sequence;
+      }
+      else
+      {
+         long sequence = al.getAndIncrement();
+
+         if (thread.isRecording())
+         {
+            thread.addSequence(sequence);
+         }
+
+      //   log.info(System.identityHashCode(this) + " got sequence non replicated " + sequence);
+
+         return sequence;
+      }
+   }
+
+   public long get()
+   {
+      return al.get();
+   }
+
+   // public long getAndIncrement(long expected)
+   // {
+   // while (!al.compareAndSet(expected, expected + 1))
+   // {
+   // Thread.yield();
+   // }
+   //
+   // return expected;
+   // }
+
+}

Modified: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicatorImpl.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicatorImpl.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/ReplicatorImpl.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -22,15 +22,17 @@
 
 package org.jboss.messaging.core.server.replication.impl;
 
-import java.util.HashSet;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Queue;
-import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
 import org.jboss.messaging.core.logging.Logger;
 import org.jboss.messaging.core.remoting.Channel;
 import org.jboss.messaging.core.remoting.Packet;
+import org.jboss.messaging.core.remoting.impl.QueuedWriteManager;
+import org.jboss.messaging.core.remoting.impl.wireformat.PacketImpl;
 import org.jboss.messaging.core.remoting.impl.wireformat.replication.ReplicateLockSequenceMessage;
 import org.jboss.messaging.core.server.replication.ReplicableAction;
 import org.jboss.messaging.core.server.replication.Replicator;
@@ -50,32 +52,69 @@
    
    private final Queue<WaitingChannelsHolder> waitingChannelsQueue = new ConcurrentLinkedQueue<WaitingChannelsHolder>();
 
-   private Set<Channel> currentChannels;
-
+   //private Set<Channel> currentChannels;
+   private Map<QueuedWriteManager, ChannelCount> currentChannelQueuedWriteCounts;
+     
    private static class WaitingChannelsHolder
    {
       Runnable postReplicateAction;
       
-      Set<Channel> channels;
+      //Set<Channel> channels;
+      Map<QueuedWriteManager, ChannelCount> channelQueuedWriteCounts;
+      
+      Packet sentPacket;
    }
-      
+   
+   private static class ChannelCount
+   {
+      int count;
+   }
+            
    public ReplicatorImpl(final Channel replicatingChannel)
    {
       this.replicatingChannel = replicatingChannel;
    }
+   
+   public void close()
+   {
+      replicatingChannel.close();
+   }
+   
+   public Channel getReplicatingChannel()
+   {
+      return replicatingChannel;
+   }
 
-   public void registerWaitingChannel(final Channel channel)
+   public void registerWaitingChannel(final QueuedWriteManager channel)
    {
-      currentChannels.add(channel);
+      ChannelCount count = currentChannelQueuedWriteCounts.get(channel);
+      
+      if (count == null)
+      {
+         count = new ChannelCount();
+         
+         currentChannelQueuedWriteCounts.put(channel, count);
+      }
+      
+      count.count++;
    }
 
    public void replicationResponseReceived()
-   {
+   {      
       WaitingChannelsHolder waitingChannelsHolder = waitingChannelsQueue.remove();
+      
+     // log.info("got replication response in replicator for sent packet " + waitingChannelsHolder.sentPacket);
             
-      for (Channel channel : waitingChannelsHolder.channels)
-      {        
-         channel.replicationResponseReceived(this);
+     // log.info("there are " + waitingChannelsHolder.channelQueuedWriteCounts.size() + " waiting channels");
+            
+//      if (waitingChannelsHolder.sentPacket.getType() == PacketImpl.SESS_CLOSE)
+//      {
+//         log.info("***Got session close response from backup");
+//      }
+      
+      for (Map.Entry<QueuedWriteManager, ChannelCount> entry: waitingChannelsHolder.channelQueuedWriteCounts.entrySet())
+      {    
+         entry.getKey().replicationResponseReceived(this, entry.getValue().count);
       }         
       
       if (waitingChannelsHolder.postReplicateAction != null)
@@ -90,8 +129,14 @@
        
       JBMThread thread = JBMThread.currentThread();
       
-      this.currentChannels = new HashSet<Channel>();
+      //sanity - in case we have nested calls to execute
+      if (thread.isRecording())
+      {
+         throw new IllegalStateException("Cannot have nested recordings");
+      }
       
+      this.currentChannelQueuedWriteCounts = new HashMap<QueuedWriteManager, ChannelCount>();
+      
       thread.setRecord(this);
       
       action.run();
@@ -105,8 +150,9 @@
       // We then send the sequences to the backup
       
       WaitingChannelsHolder holder = new WaitingChannelsHolder();
-      holder.channels = currentChannels;
+      holder.channelQueuedWriteCounts = currentChannelQueuedWriteCounts;
       holder.postReplicateAction = postReplicateAction;
+      holder.sentPacket = action.getPacket();
       
       waitingChannelsQueue.add(holder);      
 

Added: branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/SequencedLock.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/SequencedLock.java	                        (rev 0)
+++ branches/Branch_MultiThreaded_Replication/src/main/org/jboss/messaging/core/server/replication/impl/SequencedLock.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -0,0 +1,222 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.core.server.replication.impl;
+
+import java.util.Arrays;
+import java.util.Queue;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.LockSupport;
+
+import org.jboss.messaging.core.logging.Logger;
+
+/**
+ * 
+ * A SequencedLock
+ *
+ * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
+ *
+ *
+ */
+public class SequencedLock
+{
+   private static final Logger log = Logger.getLogger(SequencedLock.class);
+
+   private final Queue<QueueEntry> waiting;
+
+   private final AtomicLong currentSequence;
+
+   private Thread owner;
+
+   public SequencedLock(final long sequence)
+   {
+      waiting = new PriorityBlockingQueue<QueueEntry>();
+
+      // this.currentSequence = sequence;
+      this.currentSequence = new AtomicLong(sequence);
+   }
+
+   private synchronized void dump(final long sequence)
+   {
+      log.error(System.identityHashCode(this) + " Timed out trying to get lock, desired " +
+                sequence +
+                " current " +
+                currentSequence);
+
+      log.info("Current owner is " + owner);
+
+      log.info("Dumping entries");
+
+      Object[] sorted = waiting.toArray();
+
+      Arrays.sort(sorted);
+
+      for (Object obj : sorted)
+      {
+         QueueEntry en = (QueueEntry)obj;
+
+         log.info("seq:" + en.sequence + " thread:" + System.identityHashCode(en.thread));
+      }
+
+      try
+      {
+         Thread.sleep(1000000);
+      }
+      catch (Exception e)
+      {
+      }
+   }
+
+   private void addEntry(final QueueEntry entry)
+   {
+//      synchronized (waiting)
+//      {
+
+         waiting.add(entry);
+     // }
+   }
+
+   private QueueEntry peekEntry()
+   {
+//      synchronized (waiting)
+//      {
+         QueueEntry entry = waiting.peek();
+
+         if (entry != null && entry.sequence == currentSequence.get())
+         {
+            return entry;
+         }
+         else
+         {
+            return null;
+         }
+     // }
+
+   }
+
+   private void removeEntry()
+   {
+     // synchronized (waiting)
+     // {
+         waiting.remove();
+     // }
+   }
+
+   private final AtomicBoolean locked = new AtomicBoolean(false);
+
+   public boolean lock(final long sequence)
+   {
+      Thread currentThread = Thread.currentThread();
+
+      // log.info("Thread " + System.identityHashCode(currentThread) + " trying to lock " +
+      // System.identityHashCode(this) + " with sequence " + sequence);
+
+      QueueEntry entry = new QueueEntry(sequence, currentThread);
+
+      addEntry(entry);
+
+      boolean wasInterrupted = false;
+
+      while (true)
+      {
+         QueueEntry peeked = peekEntry();
+
+         if (peeked == null || peeked.thread != currentThread || !locked.compareAndSet(false, true))
+         {
+            LockSupport.park();
+
+            if (Thread.interrupted())
+            {
+               wasInterrupted = true;
+            }
+         }
+         else
+         {
+            break;
+         }
+      }
+
+      if (wasInterrupted)
+      {
+         currentThread.interrupt();
+      }
+
+      removeEntry();
+
+      owner = currentThread;
+
+      // log.info("Thread " + System.identityHashCode(currentThread) + " locked " + System.identityHashCode(this) + "
+      // with sequence " + sequence);
+
+      return true;
+   }
+
+   public void unlock()
+   {
+      // log.info("Thread " + System.identityHashCode(Thread.currentThread()) + " unlocking " +
+      // System.identityHashCode(this));
+
+      if (owner != Thread.currentThread())
+      {
+         throw new IllegalMonitorStateException();
+      }
+
+      currentSequence.incrementAndGet();
+
+      locked.set(false);
+
+      QueueEntry entry = peekEntry();
+
+      if (entry != null)
+      {
+         LockSupport.unpark(peekEntry().thread);
+      }
+   }
+
+   private static final class QueueEntry implements Comparable<QueueEntry>
+   {
+      private final long sequence;
+
+      private final Thread thread;
+
+      private QueueEntry(final long sequence, final Thread thread)
+      {
+         this.sequence = sequence;
+
+         if (thread == null)
+         {
+            throw new NullPointerException("Null thread");
+         }
+
+         this.thread = thread;
+      }
+
+      public int compareTo(final QueueEntry entry)
+      {
+         long l = entry.sequence;
+
+         return sequence < l ? -1 : (sequence == l ? 0 : 1);
+      }
+   }
+}

Modified: branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadFailoverSupport.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadFailoverSupport.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadFailoverSupport.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -256,25 +256,25 @@
       @Override
       public synchronized void run()
       {
-         log.info("** Failing connection");
-
-         RemotingConnectionImpl conn = (RemotingConnectionImpl)((ClientSessionImpl)session).getConnection();
-
-         if (failOnCreateConnection)
-         {
-            InVMConnector.numberOfFailures = 1;
-            InVMConnector.failOnCreateConnection = true;
-         }
-         else
-         {
-            conn.fail(new MessagingException(MessagingException.NOT_CONNECTED, "blah"));
-         }
-
-         log.info("** Fail complete");
-
-         cancel();
-
-         executed = true;
+//         log.info("** Failing connection");
+//
+//         RemotingConnectionImpl conn = (RemotingConnectionImpl)((ClientSessionImpl)session).getConnection();
+//
+//         if (failOnCreateConnection)
+//         {
+//            InVMConnector.numberOfFailures = 1;
+//            InVMConnector.failOnCreateConnection = true;
+//         }
+//         else
+//         {
+//            conn.fail(new MessagingException(MessagingException.NOT_CONNECTED, "blah"));
+//         }
+//
+//         log.info("** Fail complete");
+//
+//         cancel();
+//
+//         executed = true;
       }
 
       public synchronized boolean isExecuted()

Modified: branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/integration/cluster/failover/SimpleAutomaticFailoverTest.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/integration/cluster/failover/SimpleAutomaticFailoverTest.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/integration/cluster/failover/SimpleAutomaticFailoverTest.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -85,61 +85,68 @@
 
    public void testReplication1() throws Exception
    {
-      ClientSessionFactory sf = new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"));
+      for (int j = 0; j < 10; j++)
+      {
+         ClientSessionFactory sf = new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"));
 
-      sf.setProducerWindowSize(32 * 1024);
+         sf.setProducerWindowSize(32 * 1024);
 
-      ClientSession session = sf.createSession(false, true, true);
+         ClientSession session = sf.createSession(false, true, true);
 
-      session.createQueue(ADDRESS, ADDRESS, null, false);
+         session.createQueue(ADDRESS, ADDRESS, null, false);
 
-      ClientProducer producer = session.createProducer(ADDRESS);
+         ClientProducer producer = session.createProducer(ADDRESS);
 
-      final int numMessages = 100;
+         final int numMessages = 100;
 
-      long start = System.currentTimeMillis();
+         long start = System.currentTimeMillis();
 
-      for (int i = 0; i < numMessages; i++)
-      {
-         ClientMessage message = session.createClientMessage(JBossTextMessage.TYPE,
-                                                             false,
-                                                             0,
-                                                             System.currentTimeMillis(),
-                                                             (byte)1);
-         message.putIntProperty(new SimpleString("count"), i);
-         message.getBody().writeString("aardvarks");
-         producer.send(message);
-      }
+         for (int i = 0; i < numMessages; i++)
+         {
+            ClientMessage message = session.createClientMessage(JBossTextMessage.TYPE,
+                                                                false,
+                                                                0,
+                                                                System.currentTimeMillis(),
+                                                                (byte)1);
+            message.putIntProperty(new SimpleString("count"), i);
+            message.getBody().writeString("aardvarks");
+            producer.send(message);
+         }
 
-      // Thread.sleep(500);
+         // Thread.sleep(500);
 
-      ClientConsumer consumer = session.createConsumer(ADDRESS);
+         ClientConsumer consumer = session.createConsumer(ADDRESS);
 
-      log.info("sent messages");
+         log.info("sent messages");
 
-      session.start();
+         session.start();
 
-      log.info("Started session");
+         log.info("Started session");
 
-      for (int i = 0; i < numMessages; i++)
-      {
-         ClientMessage message2 = consumer.receive();
+         for (int i = 0; i < numMessages; i++)
+         {
+            ClientMessage message2 = consumer.receive();
 
-         assertEquals("aardvarks", message2.getBody().readString());
-         assertEquals(i, message2.getProperty(new SimpleString("count")));
+            assertEquals("aardvarks", message2.getBody().readString());
+            assertEquals(i, message2.getProperty(new SimpleString("count")));
 
-         message2.acknowledge();
-      }
+            message2.acknowledge();
+         }
 
-      long end = System.currentTimeMillis();
+         long end = System.currentTimeMillis();
 
-      log.info("That took " + (end - start));
+         log.info("That took " + (end - start));
 
-      ClientMessage message3 = consumer.receive(250);
+         ClientMessage message3 = consumer.receive(250);
 
-      assertNull(message3);
+         assertNull(message3);
 
-      session.close();
+         session.close();
+
+         tearDown();
+
+         setUp();
+      }
    }
 
    public void testReplication2() throws Exception
@@ -764,7 +771,7 @@
       for (int j = 0; j < 10; j++)
       {
          log.info("Iteration " + j);
-         
+
          ClientSessionFactoryInternal sf = new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"),
                                                                         new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
                                                                                                    backupParams));
@@ -832,62 +839,85 @@
          assertEquals(0, sf.numConnections());
 
          sf.close();
-         
+
          tearDown();
-         
+
          setUp();
       }
    }
 
    public void testFailoverWithNotifications() throws Exception
    {
-      ClientSessionFactoryInternal sf = new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"),
-                                                                     new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
-                                                                                                backupParams));
+      for (int j = 0; j < 10; j++)
+      {
+         log.info("ITERATION " + j);
+         
+         ClientSessionFactoryInternal sf = new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"),
+                                                                        new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
+                                                                                                   backupParams));
 
-      sf.setProducerWindowSize(32 * 1024);
+         sf.setProducerWindowSize(32 * 1024);
 
-      ClientSession sess = sf.createSession(false, true, true);
+         log.info("creating session");
+         
+         ClientSession sess = sf.createSession(false, true, true);
+         
+         log.info("creating notif queue");
 
-      sess.createQueue("jbm.notifications", "notifqueue", false);
+         sess.createQueue("jbm.notifications", "notifqueue", false);
+         
+         log.info("creating consumer");
 
-      ClientConsumer cons = sess.createConsumer("notifqueue");
+         ClientConsumer cons = sess.createConsumer("notifqueue");
 
-      sess.start();
+         log.info("starting session");
+         
+         sess.start();
+         
+         log.info("creating other queue");
 
-      sess.createQueue("blah", "blah", false);
-      sess.createQueue("blah", "blah2", false);
+         sess.createQueue("blah", "blah", false);
 
-      ClientMessage msg = cons.receive(1000);
-      assertNotNull(msg);
-      msg.acknowledge();
+         sess.createQueue("blah", "blah2", false);
 
-      msg = cons.receive(1000);
-      assertNotNull(msg);
-      msg.acknowledge();
+         ClientMessage msg = cons.receive(1000);
+         assertNotNull(msg);
+         msg.acknowledge();
 
-      sess.stop();
+         msg = cons.receive(1000);
+         assertNotNull(msg);
+         msg.acknowledge();
 
-      sess.createQueue("blah", "blah3", false);
-      sess.createQueue("blah", "blah4", false);
+         sess.stop();
 
-      RemotingConnection conn = ((ClientSessionImpl)sess).getConnection();
+         sess.createQueue("blah", "blah3", false);
+         sess.createQueue("blah", "blah4", false);
+         
+         log.info("** created more queues");
 
-      // Simulate failure on connection
-      conn.fail(new MessagingException(MessagingException.NOT_CONNECTED));
+         RemotingConnection conn = ((ClientSessionImpl)sess).getConnection();
 
-      sess.start();
+         // Simulate failure on connection
+         conn.fail(new MessagingException(MessagingException.NOT_CONNECTED));
 
-      msg = cons.receive(1000);
-      assertNotNull(msg);
-      msg.acknowledge();
+         sess.start();
 
-      msg = cons.receive(1000);
-      assertNotNull(msg);
-      msg.acknowledge();
+         msg = cons.receive(1000);
+         assertNotNull(msg);
+         msg.acknowledge();
 
-      sess.close();
+         msg = cons.receive(1000);
+         assertNotNull(msg);
+         msg.acknowledge();
+         
+         //Thread.sleep(500);
 
+         sess.close();
+
+         tearDown();
+
+         setUp();
+      }
    }
 
    /*

Deleted: branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/unit/core/server/replication/impl/NewSequencedLockTest.java
===================================================================
--- branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/unit/core/server/replication/impl/NewSequencedLockTest.java	2009-07-07 16:10:40 UTC (rev 7535)
+++ branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/unit/core/server/replication/impl/NewSequencedLockTest.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -1,215 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source
- * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
- * by the @authors tag. See the copyright.txt in the distribution for a
- * full listing of individual contributors.
- *
- * This is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License as
- * published by the Free Software Foundation; either version 2.1 of
- * the License, or (at your option) any later version.
- *
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this software; if not, write to the Free
- * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
- */
-
-package org.jboss.messaging.tests.unit.core.server.replication.impl;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-import junit.framework.TestCase;
-
-import org.jboss.messaging.core.logging.Logger;
-import org.jboss.messaging.core.server.replication.impl.PriorityLock;
-
-/**
- * A NewSequencedLockTest
- *
- * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
- *
- *
- */
-public class NewSequencedLockTest extends TestCase
-{
-   private static final Logger log = Logger.getLogger(NewSequencedLockTest.class);
-
-   public void testAscending() throws Exception
-   {
-      for (int i = 0; i < 1000; i++)
-      {
-         log.info("iter " + i);
-         this.doTestSequences(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-      }
-   }
-
-   public void testDescending() throws Exception
-   {
-      for (int i = 0; i < 1000; i++)
-      {
-         log.info("iter " + i);
-         this.doTestSequences(9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
-      }
-   }
-
-   public void testAlternate() throws Exception
-   {
-      for (int i = 0; i < 1000; i++)
-      {
-         log.info("iter " + i);
-         this.doTestSequences(9, 0, 8, 1, 7, 2, 6, 3, 5, 4);
-      }
-   }
-
-   public void testRandom() throws Exception
-   {
-      for (int i = 0; i < 1000; i++)
-      {
-         log.info("iter " + i);
-         this.doTestSequences(3, 9, 5, 7, 1, 2, 0, 8, 6, 4);
-      }
-   }
-
-   public void testSpeed() throws Exception
-   {
-      PriorityLock lock = new PriorityLock();
-
-      AtomicInteger counter = new AtomicInteger(0);
-      
-      final int numThreads = 10;
-      
-      final int acquires = 10000000;
-      
-      MyThread2[] threads = new MyThread2[numThreads];
-      
-      for (int i = 0; i < numThreads; i++)
-      {
-         threads[i] = new MyThread2(lock, acquires, counter);
-      }
-      
-      long start = System.currentTimeMillis();
-
-      for (int i = 0; i < numThreads; i++)
-      {
-         threads[i].start();
-      }
-
-      for (int i = 0; i < numThreads; i++)
-      {
-         threads[i].join();
-      }
-
-      long end = System.currentTimeMillis();
-      
-      double rate = 1000 * ((double)acquires * numThreads) / (end - start);
-
-      System.out.println("Rate " + rate);
-     
-   }
-
-   class MyThread2 extends Thread
-   {
-      private int acquireCount;
-      
-      private final AtomicInteger counter;
-      
-      private final PriorityLock lock;
-
-      MyThread2(final PriorityLock lock, final int acquires, final AtomicInteger counter)
-      {
-         this.lock= lock;
-         
-         this.acquireCount = acquires;
-         
-         this.counter = counter;
-      }
-      
-      public void run()
-      {
-         for (int i = 0; i < acquireCount; i++)
-         {
-            lock.lock(counter.getAndIncrement());
-            
-            lock.unlock();
-         }
-      }
-   }
-
-   private void doTestSequences(final int... sequences) throws Exception
-   {
-      PriorityLock lock = new PriorityLock();
-
-      AtomicInteger counter = new AtomicInteger(0);
-
-      MyThread[] threads = new MyThread[sequences.length];
-
-      for (int i = 0; i < sequences.length; i++)
-      {
-         threads[i] = new MyThread(sequences[i], counter, lock);
-      }
-
-      for (int i = 0; i < sequences.length; i++)
-      {
-         threads[i].start();
-      }
-
-      for (int i = 0; i < sequences.length; i++)
-      {
-         threads[i].join();
-      }
-
-      for (int i = 0; i < sequences.length; i++)
-      {
-         assertEquals(sequences[i], threads[i].getObtainedSeq());
-      }
-   }
-
-   class MyThread extends Thread
-   {
-      private AtomicInteger counter;
-
-      private PriorityLock lock;
-
-      private final int seq;
-
-      private volatile int obtainedSeq;
-
-      MyThread(final int seq, final AtomicInteger counter, final PriorityLock lock)
-      {
-         this.seq = seq;
-
-         this.counter = counter;
-
-         this.lock = lock;
-      }
-
-      public void run()
-      {
-         // log.info(System.identityHashCode(this) + " Will attempt to obtain lock with sequence " + seq);
-
-         lock.lock(seq);
-
-         // log.info(System.identityHashCode(this) + " Obtained lock with sequence " + seq);
-
-         try
-         {
-            obtainedSeq = counter.getAndIncrement();
-         }
-         finally
-         {
-            lock.unlock();
-         }
-      }
-
-      public int getObtainedSeq()
-      {
-         return this.obtainedSeq;
-      }
-   }
-}

Copied: branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/unit/core/server/replication/impl/SequencedLockTest.java (from rev 7478, branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/unit/core/server/replication/impl/NewSequencedLockTest.java)
===================================================================
--- branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/unit/core/server/replication/impl/SequencedLockTest.java	                        (rev 0)
+++ branches/Branch_MultiThreaded_Replication/tests/src/org/jboss/messaging/tests/unit/core/server/replication/impl/SequencedLockTest.java	2009-07-07 19:45:08 UTC (rev 7536)
@@ -0,0 +1,234 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.tests.unit.core.server.replication.impl;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import junit.framework.TestCase;
+
+import org.jboss.messaging.core.logging.Logger;
+import org.jboss.messaging.core.server.replication.impl.SequencedLock;
+
+/**
+ * A SequencedLockTest
+ *
+ * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
+ *
+ *
+ */
+public class SequencedLockTest extends TestCase
+{
+   private static final Logger log = Logger.getLogger(SequencedLockTest.class);
+
+   public void testAscending() throws Exception
+   {
+      for (int i = 0; i < 1000; i++)
+      {
+         //log.info("iter " + i);
+         this.doTestSequences(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+      }
+   }
+
+   public void testDescending() throws Exception
+   {
+      for (int i = 0; i < 1000; i++)
+      {
+         //log.info("iter " + i);
+         this.doTestSequences(9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
+      }
+   }
+
+   public void testAlternate() throws Exception
+   {
+      for (int i = 0; i < 1000; i++)
+      {
+        // log.info("iter " + i);
+         this.doTestSequences(9, 0, 8, 1, 7, 2, 6, 3, 5, 4);
+      }
+   }
+
+   public void testRandom() throws Exception
+   {
+      for (int i = 0; i < 1000; i++)
+      {
+        // log.info("iter " + i);
+         this.doTestSequences(3, 9, 5, 7, 1, 2, 0, 8, 6, 4);
+      }
+   }
+
+   public void testSpeed() throws Exception
+   {
+      SequencedLock lock = new SequencedLock(0);
+
+      AtomicInteger counter = new AtomicInteger(0);
+      
+      AtomicInteger obcounter = new AtomicInteger(0);
+      
+      final int numThreads = 10;
+      
+      final int acquires = 100000;
+      
+      MyThread2[] threads = new MyThread2[numThreads];
+      
+      for (int i = 0; i < numThreads; i++)
+      {
+         threads[i] = new MyThread2(lock, acquires, counter, obcounter);
+      }
+      
+      long start = System.currentTimeMillis();
+
+      for (int i = 0; i < numThreads; i++)
+      {
+         threads[i].start();
+      }
+
+      for (int i = 0; i < numThreads; i++)
+      {
+         threads[i].join();
+      }
+
+      long end = System.currentTimeMillis();
+      
+      double rate = 1000 * ((double)acquires * numThreads) / (end - start);
+
+      System.out.println("Rate " + rate);
+     
+   }
+
+   class MyThread2 extends Thread
+   {
+      private int acquireCount;
+      
+      private final AtomicInteger counter;
+      
+      private final AtomicInteger observedCounter;
+      
+      private final SequencedLock lock;
+
+      MyThread2(final SequencedLock lock, final int acquires, final AtomicInteger counter, final AtomicInteger observedCounter)
+      {
+         this.lock= lock;
+         
+         this.acquireCount = acquires;
+         
+         this.counter = counter;
+         
+         this.observedCounter = observedCounter;
+      }
+      
+      public void run()
+      {
+         for (int i = 0; i < acquireCount; i++)
+         {
+            int cnt = counter.getAndIncrement();
+            
+          //  log.info("Trying lock " + cnt);
+            
+            lock.lock(cnt);
+            
+            int ob = observedCounter.getAndIncrement();
+            
+           // log.info("Got lock " + ob);
+            
+            if (cnt != ob)
+            {
+               System.out.println("Out of order, cnt " + cnt + " ob " + ob);
+            }
+            
+            lock.unlock();
+         }
+      }
+   }
+
+   private void doTestSequences(final int... sequences) throws Exception
+   {
+      SequencedLock lock = new SequencedLock(0);
+
+      AtomicInteger counter = new AtomicInteger(0);
+
+      MyThread[] threads = new MyThread[sequences.length];
+
+      for (int i = 0; i < sequences.length; i++)
+      {
+         threads[i] = new MyThread(sequences[i], counter, lock);
+      }
+
+      for (int i = 0; i < sequences.length; i++)
+      {
+         threads[i].start();
+      }
+
+      for (int i = 0; i < sequences.length; i++)
+      {
+         threads[i].join();
+      }
+
+      for (int i = 0; i < sequences.length; i++)
+      {
+         assertEquals(sequences[i], threads[i].getObtainedSeq());
+      }
+   }
+
+   class MyThread extends Thread
+   {
+      private AtomicInteger counter;
+
+      private SequencedLock lock;
+
+      private final int seq;
+
+      private volatile int obtainedSeq;
+
+      MyThread(final int seq, final AtomicInteger counter, final SequencedLock lock)
+      {
+         this.seq = seq;
+
+         this.counter = counter;
+
+         this.lock = lock;
+      }
+
+      public void run()
+      {
+         // log.info(System.identityHashCode(this) + " Will attempt to obtain lock with sequence " + seq);
+
+         lock.lock(seq);
+
+         // log.info(System.identityHashCode(this) + " Obtained lock with sequence " + seq);
+
+         try
+         {
+            obtainedSeq = counter.getAndIncrement();
+         }
+         finally
+         {
+            lock.unlock();
+         }
+      }
+
+      public int getObtainedSeq()
+      {
+         return this.obtainedSeq;
+      }
+   }
+}




More information about the jboss-cvs-commits mailing list