[jboss-cvs] JBoss Messaging SVN: r5678 - in trunk: src/main/org/jboss/messaging/core/paging and 10 other directories.

jboss-cvs-commits at lists.jboss.org jboss-cvs-commits at lists.jboss.org
Wed Jan 21 16:43:24 EST 2009


Author: clebert.suconic at jboss.com
Date: 2009-01-21 16:43:23 -0500 (Wed, 21 Jan 2009)
New Revision: 5678

Added:
   trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/LargeMessageMuliThreadFailoverTest.java
   trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadRandomFailoverTestBase.java
Removed:
   trunk/tests/src/org/jboss/messaging/tests/integration/paging/remote/
Modified:
   trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerImpl.java
   trunk/src/main/org/jboss/messaging/core/client/impl/ClientProducerImpl.java
   trunk/src/main/org/jboss/messaging/core/paging/PagingManager.java
   trunk/src/main/org/jboss/messaging/core/paging/PagingStore.java
   trunk/src/main/org/jboss/messaging/core/paging/impl/PageTransactionInfoImpl.java
   trunk/src/main/org/jboss/messaging/core/paging/impl/PagingManagerImpl.java
   trunk/src/main/org/jboss/messaging/core/paging/impl/PagingStoreImpl.java
   trunk/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalLargeServerMessage.java
   trunk/src/main/org/jboss/messaging/core/persistence/impl/nullpm/NullStorageLargeServerMessage.java
   trunk/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionContinuationMessage.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReceiveContinuationMessage.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReceiveMessage.java
   trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReplicateDeliveryMessage.java
   trunk/src/main/org/jboss/messaging/core/server/ServerConsumer.java
   trunk/src/main/org/jboss/messaging/core/server/ServerMessage.java
   trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java
   trunk/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java
   trunk/src/main/org/jboss/messaging/core/server/impl/ServerMessageImpl.java
   trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java
   trunk/tests/src/org/jboss/messaging/tests/integration/chunkmessage/MessageChunkTest.java
   trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/FailoverTestBase.java
   trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/JustReplicationTest.java
   trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadRandomFailoverTest.java
   trunk/tests/src/org/jboss/messaging/tests/integration/paging/PageCrashTest.java
   trunk/tests/src/org/jboss/messaging/tests/integration/paging/PagingManagerIntegrationTest.java
   trunk/tests/src/org/jboss/messaging/tests/integration/paging/PagingServiceIntegrationTest.java
Log:
LargeMessage & Paging Failover (part I)

Modified: trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerImpl.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/client/impl/ClientConsumerImpl.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -344,7 +344,7 @@
       }
 
       // Flow control for the first packet, we will have others
-      flowControl(packet.getPacketSize());
+      flowControl(packet.getPacketSize(), true);
 
       currentChunkMessage = createFileMessage(packet.getLargeMessageHeader());
    }
@@ -360,7 +360,7 @@
 
       if (chunk.isContinues())
       {
-         flowControl(chunk.getPacketSize());
+         flowControl(chunk.getPacketSize(), true);
       }
 
       if (isFileConsumer())
@@ -468,7 +468,7 @@
       sessionExecutor.execute(runner);
    }
 
-   private void flowControl(final int messageBytes) throws MessagingException
+   private void flowControl(final int messageBytes, final boolean useExecutor) throws MessagingException
    {
       if (clientWindowSize > 0)
       {
@@ -476,9 +476,28 @@
 
          if (creditsToSend >= clientWindowSize)
          {
-            channel.send(new SessionConsumerFlowCreditMessage(id, creditsToSend));
 
-            creditsToSend = 0;
+            if (useExecutor)
+            {
+               // Flowcontrol on largeMessages continuations needs to be done in a separate thread or failover would block
+               final int credits = creditsToSend;
+
+               creditsToSend = 0;
+               sessionExecutor.execute(new Runnable()
+               {
+
+                  public void run()
+                  {
+                     channel.send(new SessionConsumerFlowCreditMessage(id, credits));
+                  }
+
+               });
+            }
+            else
+            {
+               channel.send(new SessionConsumerFlowCreditMessage(id, creditsToSend));
+               creditsToSend = 0;
+            }
          }
       }
    }
@@ -567,7 +586,7 @@
    private void flowControlBeforeConsumption(final ClientMessageInternal message) throws MessagingException
    {
       // Chunk messages will execute the flow control while receiving the chunks
-      flowControl(message.getFlowControlSize());
+      flowControl(message.getFlowControlSize(), false);
    }
 
    private void doCleanUp(final boolean sendCloseMessage) throws MessagingException
@@ -608,7 +627,7 @@
          {
             channel.sendBlocking(new SessionConsumerCloseMessage(id));
          }
-         
+
          clearBuffer();
       }
       finally
@@ -616,16 +635,16 @@
          session.removeConsumer(this);
       }
    }
-   
+
    private void clearBuffer()
    {
       if (isFileConsumer())
       {
-         for (ClientMessage message: buffer)
+         for (ClientMessage message : buffer)
          {
             if (message instanceof ClientFileMessage)
             {
-               ((ClientFileMessage) message).getFile().delete();
+               ((ClientFileMessage)message).getFile().delete();
             }
          }
       }
@@ -675,7 +694,7 @@
                                                   "-" +
                                                   getID() +
                                                   ".jbm"));
-         
+
          cloneMessage.setFlowControlSize(message.getFlowControlSize());
 
          addBytesBody(cloneMessage, message.getBody().array());
@@ -696,7 +715,7 @@
          if (!directory.exists())
          {
             boolean ok = directory.mkdirs();
-            
+
             if (!ok)
             {
                throw new IOException("Failed to create directory " + directory.getCanonicalPath());

Modified: trunk/src/main/org/jboss/messaging/core/client/impl/ClientProducerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/client/impl/ClientProducerImpl.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/client/impl/ClientProducerImpl.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -258,7 +258,7 @@
       if (headerSize > minLargeMessageSize)
       {
          throw new MessagingException(MessagingException.ILLEGAL_STATE,
-                                      "Header size is too big, use the messageBody for large data, or increase minLargeMessageSize");
+                                      "Header size (" + headerSize + ") is too big, use the messageBody for large data, or increase minLargeMessageSize");
       }
 
       MessagingBuffer headerBuffer = new ByteBufferWrapper(ByteBuffer.allocate(headerSize));

Modified: trunk/src/main/org/jboss/messaging/core/paging/PagingManager.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/paging/PagingManager.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/paging/PagingManager.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -58,6 +58,10 @@
  */
 public interface PagingManager extends MessagingComponent
 {
+   void activate();
+   
+   boolean isBackup();
+   
    /** The system is paging because of global-page-mode */
    boolean isGlobalPageMode();
 

Modified: trunk/src/main/org/jboss/messaging/core/paging/PagingStore.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/paging/PagingStore.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/paging/PagingStore.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -62,6 +62,8 @@
    void sync() throws Exception;
 
    boolean page(PagedMessage message, boolean sync, boolean duplicateDetection) throws Exception;
+   
+   public boolean readPage() throws Exception;
 
    /**
     * 

Modified: trunk/src/main/org/jboss/messaging/core/paging/impl/PageTransactionInfoImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/paging/impl/PageTransactionInfoImpl.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/paging/impl/PageTransactionInfoImpl.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -47,7 +47,7 @@
 
    private volatile long recordID;
 
-   private CountDownLatch countDownCompleted;
+   private volatile CountDownLatch countDownCompleted;
 
    private volatile boolean committed;
    private volatile boolean rolledback;

Modified: trunk/src/main/org/jboss/messaging/core/paging/impl/PagingManagerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/paging/impl/PagingManagerImpl.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/paging/impl/PagingManagerImpl.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -36,7 +36,6 @@
 import org.jboss.messaging.core.paging.PagingStoreFactory;
 import org.jboss.messaging.core.persistence.StorageManager;
 import org.jboss.messaging.core.postoffice.PostOffice;
-import org.jboss.messaging.core.server.MessageReference;
 import org.jboss.messaging.core.server.ServerMessage;
 import org.jboss.messaging.core.settings.HierarchicalRepository;
 import org.jboss.messaging.core.settings.impl.QueueSettings;
@@ -59,6 +58,8 @@
    private volatile boolean started = false;
 
    private final long maxGlobalSize;
+   
+   private volatile boolean backup;
 
    private final AtomicLong globalSize = new AtomicLong(0);
 
@@ -91,7 +92,8 @@
                             final HierarchicalRepository<QueueSettings> queueSettingsRepository,
                             final long maxGlobalSize,
                             final long defaultPageSize,
-                            final boolean syncNonTransactional)
+                            final boolean syncNonTransactional,
+                            final boolean backup)
    {
       this.pagingStoreFactory = pagingSPI;
       this.queueSettingsRepository = queueSettingsRepository;
@@ -99,6 +101,7 @@
       this.defaultPageSize = defaultPageSize;
       this.maxGlobalSize = maxGlobalSize;
       this.syncNonTransactional = syncNonTransactional;
+      this.backup = backup; 
    }
 
    // Public
@@ -107,6 +110,19 @@
    // PagingManager implementation
    // -----------------------------------------------------------------------------------------------------
 
+   public void activate()
+   {
+      this.backup = false;
+      
+      startGlobalDepage();
+   }
+   
+   
+   public boolean isBackup()
+   {
+      return this.backup;
+   }
+   
    public boolean isGlobalPageMode()
    {
       return globalMode.get();
@@ -278,10 +294,13 @@
 
    public synchronized void startGlobalDepage()
    {
-      setGlobalPageMode(true);
-      for (PagingStore store : stores.values())
+      if (!isBackup())
       {
-         store.startDepaging(pagingStoreFactory.getGlobalDepagerExecutor());
+         setGlobalPageMode(true);
+         for (PagingStore store : stores.values())
+         {
+            store.startDepaging(pagingStoreFactory.getGlobalDepagerExecutor());
+         }
       }
    }
 

Modified: trunk/src/main/org/jboss/messaging/core/paging/impl/PagingStoreImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/paging/impl/PagingStoreImpl.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/paging/impl/PagingStoreImpl.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -46,6 +46,7 @@
 import org.jboss.messaging.core.paging.PagingStoreFactory;
 import org.jboss.messaging.core.persistence.StorageManager;
 import org.jboss.messaging.core.postoffice.PostOffice;
+import org.jboss.messaging.core.server.SendLock;
 import org.jboss.messaging.core.server.ServerMessage;
 import org.jboss.messaging.core.settings.impl.QueueSettings;
 import org.jboss.messaging.core.transaction.Transaction;
@@ -363,17 +364,17 @@
             // We set the duplicate detection header to prevent the message being depaged more than once in case of
             // failure during depage
 
+            ServerMessage msg = message.getMessage(storageManager);
+
             byte[] bytes = new byte[8];
 
             ByteBuffer buff = ByteBuffer.wrap(bytes);
 
-            ServerMessage msg = message.getMessage(storageManager);
-
             buff.putLong(msg.getMessageID());
 
             SimpleString duplID = new SimpleString(bytes);
 
-            message.getMessage(storageManager).putStringProperty(MessageImpl.HDR_DUPLICATE_DETECTION_ID, duplID);
+            msg.putStringProperty(MessageImpl.HDR_DUPLICATE_DETECTION_ID, duplID);
          }
 
          int bytesToWrite = message.getEncodeSize() + PageImpl.SIZE_RECORD;
@@ -450,6 +451,11 @@
 
    public boolean startDepaging(final Executor executor)
    {
+      if (pagingManager.isBackup())
+      {
+         return false;
+      }
+
       currentPageLock.readLock().lock();
       try
       {
@@ -631,6 +637,38 @@
       }
    }
 
+   /**
+    * Depage one page-file, read it and send it to the pagingManager / postoffice
+    * @return
+    * @throws Exception
+    */
+   public boolean readPage() throws Exception
+   {
+      Page page = depage();
+
+      if (page == null)
+      {
+         return false;
+      }
+
+      page.open();
+
+      List<PagedMessage> messages = page.read();
+
+      onDepage(page.getPageId(), storeName, messages);
+
+      if (onDepage(page.getPageId(), storeName, messages))
+      {
+         page.delete();
+         return true;
+      }
+      else
+      {
+         return false;
+      }
+
+   }
+
    // TestSupportPageStore ------------------------------------------
 
    public void forceAnotherPage() throws Exception
@@ -641,6 +679,7 @@
    /** 
     *  It returns a Page out of the Page System without reading it. 
     *  The method calling this method will remove the page and will start reading it outside of any locks.
+    *  This method could also replace the current file by a new file, and that process is done through acquiring a writeLock on currentPageLock
     *   
     *  Observation: This method is used internally as part of the regular depage process, but externally is used only on tests, 
     *               and that's why this method is part of the Testable Interface 
@@ -778,102 +817,113 @@
       // back to where it was
 
       Transaction depageTransaction = new TransactionImpl(storageManager);
-
-      depageTransaction.putProperty(TransactionPropertyIndexes.IS_DEPAGE, Boolean.valueOf(true));
-
-      HashSet<PageTransactionInfo> pageTransactionsToUpdate = new HashSet<PageTransactionInfo>();
       
-      for (PagedMessage pagedMessage : pagedMessages)
+      SendLock sendLock = postOffice.getAddressLock(destination);
+      
+      sendLock.beforeSend();
+      
+      try
       {
-         ServerMessage message = null;
-
-         message = pagedMessage.getMessage(storageManager);
-
-         final long transactionIdDuringPaging = pagedMessage.getTransactionID();
-
-         if (transactionIdDuringPaging >= 0)
+         depageTransaction.putProperty(TransactionPropertyIndexes.IS_DEPAGE, Boolean.valueOf(true));
+   
+         HashSet<PageTransactionInfo> pageTransactionsToUpdate = new HashSet<PageTransactionInfo>();
+         
+         for (PagedMessage pagedMessage : pagedMessages)
          {
-            final PageTransactionInfo pageTransactionInfo = pagingManager.getTransaction(transactionIdDuringPaging);
-
-            // http://wiki.jboss.org/wiki/JBossMessaging2Paging
-            // This is the Step D described on the "Transactions on Paging"
-            // section
-            if (pageTransactionInfo == null)
+            ServerMessage message = null;
+   
+            message = pagedMessage.getMessage(storageManager);
+   
+            final long transactionIdDuringPaging = pagedMessage.getTransactionID();
+   
+            if (transactionIdDuringPaging >= 0)
             {
-               log.warn("Transaction " + pagedMessage.getTransactionID() +
-                        " used during paging not found, ignoring message " +
-                        message);
-
-               continue;
-            }
-
-            // This is to avoid a race condition where messages are depaged
-            // before the commit arrived
-            
-            while (running && !pageTransactionInfo.waitCompletion(500))
-            {
-               // This is just to give us a chance to interrupt the process..
-               // if we start a shutdown in the middle of transactions, the commit/rollback may never come, delaying the shutdown of the server
-               if (isTrace)
+               final PageTransactionInfo pageTransactionInfo = pagingManager.getTransaction(transactionIdDuringPaging);
+   
+               // http://wiki.jboss.org/wiki/JBossMessaging2Paging
+               // This is the Step D described on the "Transactions on Paging"
+               // section
+               if (pageTransactionInfo == null)
                {
-                  trace("Waiting pageTransaction to complete");
+                  log.warn("Transaction " + pagedMessage.getTransactionID() +
+                           " used during paging not found, ignoring message " +
+                           message);
+   
+                  continue;
                }
+   
+               // This is to avoid a race condition where messages are depaged
+               // before the commit arrived
+               
+               while (running && !pageTransactionInfo.waitCompletion(500))
+               {
+                  // This is just to give us a chance to interrupt the process..
+                  // if we start a shutdown in the middle of transactions, the commit/rollback may never come, delaying the shutdown of the server
+                  if (isTrace)
+                  {
+                     trace("Waiting pageTransaction to complete");
+                  }
+               }
+               
+               if (!running)
+               {
+                  break;
+               }
+               
+               if (!pageTransactionInfo.isCommit())
+               {
+                  if (isTrace)
+                  {
+                     trace("Rollback was called after prepare, ignoring message " + message);
+                  }
+                  continue;
+               }
+   
+               // Update information about transactions
+               if (message.isDurable())
+               {
+                  pageTransactionInfo.decrement();
+                  pageTransactionsToUpdate.add(pageTransactionInfo);
+               }
             }
-            
-            if (!running)
+   
+            postOffice.route(message, depageTransaction);
+         }
+         
+         if (!running)
+         {
+            depageTransaction.rollback();
+            return false;
+         }
+   
+         for (PageTransactionInfo pageWithTransaction : pageTransactionsToUpdate)
+         {
+            // This will set the journal transaction to commit;
+            depageTransaction.putProperty(TransactionPropertyIndexes.CONTAINS_PERSISTENT, true);
+   
+            if (pageWithTransaction.getNumberOfMessages() == 0)
             {
-               break;
+               // http://wiki.jboss.org/wiki/JBossMessaging2Paging
+               // numberOfReads==numberOfWrites -> We delete the record
+               storageManager.deletePageTransactional(depageTransaction.getID(), pageWithTransaction.getRecordID());
+               pagingManager.removeTransaction(pageWithTransaction.getTransactionID());
             }
-            
-            if (!pageTransactionInfo.isCommit())
+            else
             {
-               if (isTrace)
-               {
-                  trace("Rollback was called after prepare, ignoring message " + message);
-               }
-               continue;
+               storageManager.storePageTransaction(depageTransaction.getID(), pageWithTransaction);
             }
-
-            // Update information about transactions
-            if (message.isDurable())
-            {
-               pageTransactionInfo.decrement();
-               pageTransactionsToUpdate.add(pageTransactionInfo);
-            }
          }
-
-         postOffice.route(message, depageTransaction);
-      }
-      
-      if (!running)
-      {
-         depageTransaction.rollback();
-         return false;
-      }
-
-      for (PageTransactionInfo pageWithTransaction : pageTransactionsToUpdate)
-      {
-         // This will set the journal transaction to commit;
-         depageTransaction.putProperty(TransactionPropertyIndexes.CONTAINS_PERSISTENT, true);
-
-         if (pageWithTransaction.getNumberOfMessages() == 0)
+   
+         depageTransaction.commit();
+   
+         if (isTrace)
          {
-            // http://wiki.jboss.org/wiki/JBossMessaging2Paging
-            // numberOfReads==numberOfWrites -> We delete the record
-            storageManager.deletePageTransactional(depageTransaction.getID(), pageWithTransaction.getRecordID());
-            pagingManager.removeTransaction(pageWithTransaction.getTransactionID());
+            trace("Depage committed, running = " + running);
          }
-         else
-         {
-            storageManager.storePageTransaction(depageTransaction.getID(), pageWithTransaction);
-         }
       }
-
-      depageTransaction.commit();
-
-      if (isTrace)
+      finally
       {
-         trace("Depage committed, running = " + running);
+         sendLock.afterSend();
       }
       
       return true;
@@ -974,30 +1024,6 @@
       return Integer.parseInt(fileName.substring(0, fileName.indexOf('.')));
    }
 
-   /**
-    * Depage one page-file, read it and send it to the pagingManager / postoffice
-    * @return
-    * @throws Exception
-    */
-   private void readPage() throws Exception
-   {
-      Page page = depage();
-
-      if (page == null)
-      {
-         return;
-      }
-
-      page.open();
-
-      List<PagedMessage> messages = page.read();
-
-      if (onDepage(page.getPageId(), storeName, messages))
-      {
-         page.delete();
-      }
-   }
-
    // Inner classes -------------------------------------------------
 
    private class DepageRunnable implements Runnable

Modified: trunk/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalLargeServerMessage.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalLargeServerMessage.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalLargeServerMessage.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -184,8 +184,13 @@
       return currentRefCount;
    }
 
-   public void deleteFile() throws MessagingException
+   public boolean isLargeMessage()
    {
+      return true;
+   }
+
+   public synchronized void deleteFile() throws MessagingException
+   {
       if (file != null)
       {
          storageManager.deleteFile(file);

Modified: trunk/src/main/org/jboss/messaging/core/persistence/impl/nullpm/NullStorageLargeServerMessage.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/persistence/impl/nullpm/NullStorageLargeServerMessage.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/persistence/impl/nullpm/NullStorageLargeServerMessage.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -103,6 +103,11 @@
       
    }
 
+   public boolean isLargeMessage()
+   {
+      return true;
+   }
+
    // Package protected ---------------------------------------------
 
    // Protected -----------------------------------------------------

Modified: trunk/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/postoffice/impl/PostOfficeImpl.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -385,6 +385,8 @@
    public List<Queue> activate()
    {
       backup = false;
+      
+      pagingManager.activate();
 
       Map<SimpleString, Binding> nameMap = addressManager.getBindings();
 

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionContinuationMessage.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionContinuationMessage.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionContinuationMessage.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -39,19 +39,20 @@
 
    // Constants -----------------------------------------------------
 
+   public static final int SESSION_CONTINUATION_BASE_SIZE = BASIC_PACKET_SIZE + DataConstants.SIZE_INT +
+                                                            DataConstants.SIZE_BOOLEAN;
+
    // Attributes ----------------------------------------------------
 
-   private byte[] body;
+   protected byte[] body;
 
-   private boolean continues;
+   protected boolean continues;
 
    // Static --------------------------------------------------------
 
    // Constructors --------------------------------------------------
 
-   public SessionContinuationMessage(byte type,
-                                     final byte[] body,
-                                     final boolean continues)
+   public SessionContinuationMessage(byte type, final byte[] body, final boolean continues)
    {
       super(type);
       this.body = body;
@@ -84,9 +85,7 @@
    @Override
    public int getRequiredBufferSize()
    {
-      return BASIC_PACKET_SIZE + DataConstants.SIZE_INT +
-             body.length +
-             DataConstants.SIZE_BOOLEAN;
+      return SESSION_CONTINUATION_BASE_SIZE + body.length; 
    }
 
    @Override

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReceiveContinuationMessage.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReceiveContinuationMessage.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReceiveContinuationMessage.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -39,6 +39,8 @@
 
    // Constants -----------------------------------------------------
 
+   public static final int SESSION_RECEIVE_CONTINUATION_BASE_SIZE = SESSION_CONTINUATION_BASE_SIZE + DataConstants.SIZE_LONG;
+   
    // Attributes ----------------------------------------------------
 
    private long consumerID;
@@ -83,7 +85,7 @@
    @Override
    public int getRequiredBufferSize()
    {
-      return super.getRequiredBufferSize() + DataConstants.SIZE_LONG;
+      return SESSION_RECEIVE_CONTINUATION_BASE_SIZE + body.length; 
    }
 
    @Override

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReceiveMessage.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReceiveMessage.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReceiveMessage.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -22,7 +22,6 @@
 
 package org.jboss.messaging.core.remoting.impl.wireformat;
 
-import org.jboss.messaging.core.client.ClientMessage;
 import org.jboss.messaging.core.client.impl.ClientMessageImpl;
 import org.jboss.messaging.core.client.impl.ClientMessageInternal;
 import org.jboss.messaging.core.logging.Logger;
@@ -41,6 +40,11 @@
 {
    // Constants -----------------------------------------------------
 
+   public static final int SESSION_RECEIVE_MESSAGE_LARGE_MESSAGE_SIZE = BASIC_PACKET_SIZE + DataConstants.SIZE_LONG +
+                                                                       DataConstants.SIZE_INT +
+                                                                       DataConstants.SIZE_BOOLEAN +
+                                                                       DataConstants.SIZE_INT;
+
    private static final Logger log = Logger.getLogger(SessionReceiveMessage.class);
 
    // Attributes ----------------------------------------------------
@@ -115,7 +119,7 @@
    {
       return largeMessageHeader;
    }
-   
+
    /**
     * @return the largeMessage
     */
@@ -133,11 +137,7 @@
    {
       if (largeMessage)
       {
-         return BASIC_PACKET_SIZE + DataConstants.SIZE_LONG +
-                DataConstants.SIZE_INT +
-                DataConstants.SIZE_BOOLEAN +
-                DataConstants.SIZE_INT +
-                largeMessageHeader.length;
+         return SESSION_RECEIVE_MESSAGE_LARGE_MESSAGE_SIZE + largeMessageHeader.length;
       }
       else
       {
@@ -147,7 +147,6 @@
                 serverMessage.getEncodeSize();
       }
    }
-
    public void encodeBody(final MessagingBuffer buffer)
    {
       buffer.putLong(consumerID);

Modified: trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReplicateDeliveryMessage.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReplicateDeliveryMessage.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/remoting/impl/wireformat/SessionReplicateDeliveryMessage.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -13,6 +13,8 @@
 package org.jboss.messaging.core.remoting.impl.wireformat;
 
 import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
+import org.jboss.messaging.util.DataConstants;
+import org.jboss.messaging.util.SimpleString;
 
 /**
  * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
@@ -28,17 +30,21 @@
 
    private long messageID;
 
+   private SimpleString address;
+
    // Static --------------------------------------------------------
 
    // Constructors --------------------------------------------------
 
-   public SessionReplicateDeliveryMessage(final long consumerID, final long messageID)
+   public SessionReplicateDeliveryMessage(final long consumerID, final long messageID, final SimpleString address)
    {
       super(SESS_REPLICATE_DELIVERY);
 
       this.consumerID = consumerID;
 
       this.messageID = messageID;
+
+      this.address = address;
    }
 
    public SessionReplicateDeliveryMessage()
@@ -58,11 +64,18 @@
       return messageID;
    }
 
+   public SimpleString getAddress()
+   {
+      return address;
+   }
+
    public void encodeBody(final MessagingBuffer buffer)
    {
       buffer.putLong(consumerID);
 
       buffer.putLong(messageID);
+
+      buffer.putSimpleString(address);
    }
 
    public void decodeBody(final MessagingBuffer buffer)
@@ -70,13 +83,20 @@
       consumerID = buffer.getLong();
 
       messageID = buffer.getLong();
+
+      address = buffer.getSimpleString();
    }
-   
+
    public boolean isRequiresConfirmations()
-   {      
+   {
       return false;
    }
 
+   public int getRequiredBufferSize()
+   {
+      return BASIC_PACKET_SIZE + DataConstants.SIZE_LONG + DataConstants.SIZE_LONG + SimpleString.sizeofString(address);
+   }
+
    public boolean equals(Object other)
    {
       if (other instanceof SessionReplicateDeliveryMessage == false)
@@ -88,7 +108,7 @@
 
       return super.equals(other) && this.consumerID == r.consumerID && this.messageID == r.messageID;
    }
-   
+
    // Package protected ---------------------------------------------
 
    // Protected -----------------------------------------------------

Modified: trunk/src/main/org/jboss/messaging/core/server/ServerConsumer.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/ServerConsumer.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/server/ServerConsumer.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -26,6 +26,7 @@
 
 import org.jboss.messaging.core.remoting.Packet;
 import org.jboss.messaging.core.transaction.Transaction;
+import org.jboss.messaging.util.SimpleString;
 
 /**
  * 
@@ -56,7 +57,7 @@
 	
 	void failedOver();
 	
-	void deliverReplicated(final long messageID) throws Exception;
+	void deliverReplicated(SimpleString address, long messageID) throws Exception;
 	
 	void lock();
 	

Modified: trunk/src/main/org/jboss/messaging/core/server/ServerMessage.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/ServerMessage.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/server/ServerMessage.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -18,7 +18,7 @@
  * License along with this software; if not, write to the Free
  * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
- */ 
+ */
 
 package org.jboss.messaging.core.server;
 
@@ -34,27 +34,28 @@
  *
  */
 public interface ServerMessage extends Message, EncodingSupport
-{      
+{
    void setMessageID(long id);
 
    MessageReference createReference(Queue queue);
 
    int incrementRefCount();
-   
+
    int incrementDurableRefCount();
-   
+
    int decrementDurableRefCount();
-   
+
    int decrementRefCount();
-   
+
    ServerMessage copy();
-   
+
    int getMemoryEstimate();
 
+   boolean isLargeMessage();
+
    void setStored();
-   
+
    boolean isStored();
-   
+
    int getRefCount();
 }
-

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/MessagingServerImpl.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -761,7 +761,8 @@
                                    queueSettingsRepository,
                                    configuration.getPagingMaxGlobalSizeBytes(),
                                    configuration.getPagingDefaultSize(),
-                                   configuration.isJournalSyncNonTransactional());
+                                   configuration.isJournalSyncNonTransactional(),
+                                   configuration.isBackup());
    }
 
    // Private

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/ServerConsumerImpl.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -26,6 +26,7 @@
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
@@ -33,8 +34,9 @@
 import org.jboss.messaging.core.exception.MessagingException;
 import org.jboss.messaging.core.filter.Filter;
 import org.jboss.messaging.core.logging.Logger;
+import org.jboss.messaging.core.paging.PagingManager;
+import org.jboss.messaging.core.paging.PagingStore;
 import org.jboss.messaging.core.persistence.StorageManager;
-import org.jboss.messaging.core.postoffice.PostOffice;
 import org.jboss.messaging.core.remoting.Channel;
 import org.jboss.messaging.core.remoting.DelayedResult;
 import org.jboss.messaging.core.remoting.Packet;
@@ -52,10 +54,9 @@
 import org.jboss.messaging.core.server.ServerConsumer;
 import org.jboss.messaging.core.server.ServerMessage;
 import org.jboss.messaging.core.server.ServerSession;
-import org.jboss.messaging.core.settings.HierarchicalRepository;
-import org.jboss.messaging.core.settings.impl.QueueSettings;
 import org.jboss.messaging.core.transaction.Transaction;
 import org.jboss.messaging.core.transaction.impl.TransactionImpl;
+import org.jboss.messaging.util.SimpleString;
 
 /**
  * Concrete implementation of a ClientConsumer.
@@ -68,19 +69,22 @@
  */
 public class ServerConsumerImpl implements ServerConsumer
 {
-   // Constants
-   // ------------------------------------------------------------------------------------
+   // Constants ------------------------------------------------------------------------------------
 
    private static final Logger log = Logger.getLogger(ServerConsumerImpl.class);
 
-   // Static
-   // ---------------------------------------------------------------------------------------
+   // Static ---------------------------------------------------------------------------------------
 
-   // Attributes
-   // -----------------------------------------------------------------------------------
+   // private static final boolean trace = log.isTraceEnabled();
+   private static final boolean trace = false;
 
-   private final boolean trace = log.isTraceEnabled();
+   private static void trace(final String message)
+   {
+      log.trace(message);
+   }
 
+   // Attributes -----------------------------------------------------------------------------------
+
    private final long id;
 
    private final Queue messageQueue;
@@ -91,20 +95,28 @@
 
    private final ServerSession session;
 
+   private final Executor executor;
+
    private final Lock lock = new ReentrantLock();
 
    private AtomicInteger availableCredits = new AtomicInteger(0);
 
    private boolean started;
 
-   private volatile LargeMessageSender largeMessageSender = null;
+   private volatile LargeMessageDeliverer largeMessageDeliverer = null;
 
+   // We will only be sending one largeMessage at any time, however during replication you may have
+   // more than one LargeMessage pending on the replicationBuffer
+   private final AtomicInteger pendingLargeMessagesCounter = new AtomicInteger(0);
+
    /**
     * if we are a browse only consumer we don't need to worry about acknowledgemenets or being started/stopeed by the session.
     */
    private final boolean browseOnly;
 
    private final StorageManager storageManager;
+   
+   private final PagingManager pagingManager;
 
    private final java.util.Queue<MessageReference> deliveringRefs = new ConcurrentLinkedQueue<MessageReference>();
 
@@ -114,8 +126,7 @@
 
    private final boolean preAcknowledge;
 
-   // Constructors
-   // ---------------------------------------------------------------------------------
+   // Constructors ---------------------------------------------------------------------------------
 
    public ServerConsumerImpl(final long id,
                              final ServerSession session,
@@ -124,8 +135,10 @@
                              final boolean started,
                              final boolean browseOnly,
                              final StorageManager storageManager,
+                             final PagingManager pagingManager,
                              final Channel channel,
-                             final boolean preAcknowledge)
+                             final boolean preAcknowledge,
+                             final Executor executor)
    {
       this.id = id;
 
@@ -135,6 +148,8 @@
 
       this.session = session;
 
+      this.executor = executor;
+
       this.started = browseOnly || started;
 
       this.browseOnly = browseOnly;
@@ -144,6 +159,8 @@
       this.channel = channel;
 
       this.preAcknowledge = preAcknowledge;
+      
+      this.pagingManager = pagingManager;
 
       messageQueue.addConsumer(this);
 
@@ -238,17 +255,17 @@
       Iterator<MessageReference> iter = refs.iterator();
 
       closed = true;
-      
+
       Transaction tx = new TransactionImpl(storageManager);
 
       while (iter.hasNext())
       {
          MessageReference ref = iter.next();
-         
-         //ref.cancel(tx, storageManager, postOffice, queueSettingsRepository);  
+
+         // ref.cancel(tx, storageManager, postOffice, queueSettingsRepository);
          ref.getQueue().cancel(tx, ref);
       }
-      
+
       tx.rollback();
    }
 
@@ -344,7 +361,7 @@
          else
          {
             ref.getQueue().acknowledge(tx, ref);
-            //ref.acknowledge(tx, storageManager, postOffice, queueSettingsRepository);
+            // ref.acknowledge(tx, storageManager, postOffice, queueSettingsRepository);
 
             // Del count is not actually updated in storage unless it's
             // cancelled
@@ -393,11 +410,11 @@
       return ref;
    }
 
-   public void deliverReplicated(final long messageID) throws Exception
+   public void deliverReplicated(final SimpleString address, final long messageID) throws Exception
    {
       // It may not be the first in the queue - since there may be multiple producers
       // sending to the queue
-      MessageReference ref = messageQueue.removeReferenceWithID(messageID);
+      MessageReference ref = removeReferenceOnBackup(address, messageID);
 
       if (ref == null)
       {
@@ -412,7 +429,9 @@
 
       if (handled != HandleStatus.HANDLED)
       {
-         throw new IllegalStateException("Reference was not handled " + ref + " " + handled);
+         throw new IllegalStateException("Reference " + ref +
+                                         " was not handled on backup node, handleStatus = " +
+                                         handled);
       }
    }
 
@@ -437,25 +456,86 @@
       lock.unlock();
    }
 
-   // Public
-   // -----------------------------------------------------------------------------
+   // Public ---------------------------------------------------------------------------------------
 
-   // Private
-   // --------------------------------------------------------------------------------------
+   // Private --------------------------------------------------------------------------------------
 
+   private MessageReference removeReferenceOnBackup(SimpleString address, long id) throws Exception
+   {
+
+      // most of the times, the remove will work ok, so we first try it without any locks
+      MessageReference ref = messageQueue.removeReferenceWithID(id);
+
+      if (ref == null)
+      {
+         PagingStore store = pagingManager.getPageStore(address);
+
+         for (;;)
+         {
+            // Can't have the same store being depaged in more than one thread
+            synchronized (store)
+            {
+               // as soon as it gets the lock, it needs to verify if another thread couldn't find the reference
+               ref = messageQueue.removeReferenceWithID(id);
+               if (ref == null)
+               {
+                  // force a depage
+                  if (!store.readPage())
+                  {
+                     break;
+                  }
+               }
+               else
+               {
+                  break;
+               }
+            }
+         }
+      }
+
+      return ref;
+
+   }
+
    private void promptDelivery()
    {
-      if (largeMessageSender != null)
+      lock.lock();
+      try
       {
-         if (largeMessageSender.sendLargeMessage())
+         // largeMessageDeliverer is aways set inside a lock
+         // if we don't acquire a lock, we will have NPE eventually
+         if (largeMessageDeliverer != null)
          {
-            // prompt Delivery only if chunk was finished
+            resumeLargeMessage();
+         }
+         else
+         {
             session.promptDelivery(messageQueue);
          }
       }
+      finally
+      {
+         lock.unlock();
+      }
+   }
+
+   /**
+    * 
+    */
+   private void resumeLargeMessage()
+   {
+      if (messageQueue.isBackup())
+      {
+         // We are supposed to finish largeMessageDeliverer, or use all the possible credits before we return this
+         // method.
+         // If we play the commands on a different order than how they were generated on the live node, we will
+         // eventually still be running this largeMessage before the next message come, what would reject messages
+         // from the cluster
+         largeMessageDeliverer.deliver();
+      }
       else
       {
-         session.promptDelivery(messageQueue);
+         executor.execute(resumeLargeMessageRunnable);
       }
    }
 
@@ -465,17 +545,11 @@
       {
          return HandleStatus.BUSY;
       }
-      
+
       lock.lock();
 
       try
       {
-         // If there is a pendingLargeMessage we can't take another message
-         // This has to be checked inside the lock as the set to null is done inside the lock
-         if (largeMessageSender != null)
-         {
-            return HandleStatus.BUSY;
-         }
 
          // If the consumer is stopped then we don't accept the message, it
          // should go back into the
@@ -485,6 +559,20 @@
             return HandleStatus.BUSY;
          }
 
+         // If there is a pendingLargeMessage we can't take another message
+         // This has to be checked inside the lock as the set to null is done inside the lock
+         if (pendingLargeMessagesCounter.get() > 0)
+         {
+            if (messageQueue.isBackup())
+            {
+               log.warn("doHandle: rejecting message while send is pending, ignoring reference = " + ref +
+                        " backup = " +
+                        messageQueue.isBackup());
+            }
+
+            return HandleStatus.BUSY;
+         }
+
          final ServerMessage message = ref.getMessage();
 
          if (filter != null && !filter.match(message))
@@ -498,46 +586,25 @@
             {
                deliveringRefs.add(ref);
             }
-            
+
             ref.getQueue().referenceHandled();
          }
-         
+
          if (preAcknowledge)
          {
-            //With pre-ack, we ack *before* sending to the client
+            // With pre-ack, we ack *before* sending to the client
             ref.getQueue().acknowledge(ref);
          }
-                  
-         // TODO: get rid of the instanceof by something like message.isLargeMessage()
-         if (message instanceof LargeServerMessage)
-         {            
-            //FIXME - please put the replication logic in the sendLargeMessage method
-            
-            DelayedResult result = channel.replicatePacket(new SessionReplicateDeliveryMessage(id,
-                                                                                               message.getMessageID()));
 
-            if (result == null)
-            {
-               sendLargeMessage(ref, message);
-            }
-            else
-            {
-               // Send when replicate delivery response comes back
-               result.setResultRunner(new Runnable()
-               {
-                  public void run()
-                  {
-                     sendLargeMessage(ref, message);
-                  }
-               });
-            }
-
+         if (message.isLargeMessage())
+         {
+            deliverLargeMessage(ref, message);
          }
          else
          {
-            sendStandardMessage(ref, message);
+            deliverStandardMessage(ref, message);
          }
-         
+
          return HandleStatus.HANDLED;
       }
       finally
@@ -546,18 +613,54 @@
       }
    }
 
-   private void sendLargeMessage(final MessageReference ref, final ServerMessage message)
+   private void deliverLargeMessage(final MessageReference ref, final ServerMessage message)
    {
-      largeMessageSender = new LargeMessageSender((LargeServerMessage)message, ref);
+      pendingLargeMessagesCounter.incrementAndGet();
 
-      largeMessageSender.sendLargeMessage();
+      final LargeMessageDeliverer localDeliverer = new LargeMessageDeliverer((LargeServerMessage)message, ref);
+
+      DelayedResult result = channel.replicatePacket(new SessionReplicateDeliveryMessage(id,
+                                                                                         message.getMessageID(),
+                                                                                         message.getDestination()));
+
+      if (result == null)
+      {
+         // it doesn't need lock because deliverLargeMesasge is already inside the lock.lock()
+         largeMessageDeliverer = localDeliverer;
+         largeMessageDeliverer.deliver();
+      }
+      else
+      {
+         result.setResultRunner(new Runnable()
+         {
+            public void run()
+            {
+               // setting & unsetting largeMessageDeliver is done inside the lock,
+               // so this needs to be locked
+               lock.lock();
+               try
+               {
+                  largeMessageDeliverer = localDeliverer;
+                  if (largeMessageDeliverer.deliver())
+                  {
+                     promptDelivery();
+                  }
+               }
+               finally
+               {
+                  lock.unlock();
+               }
+            }
+         });
+      }
+
    }
 
    /**
     * @param ref
     * @param message
     */
-   private void sendStandardMessage(final MessageReference ref, final ServerMessage message)
+   private void deliverStandardMessage(final MessageReference ref, final ServerMessage message)
    {
       if (availableCredits != null)
       {
@@ -566,7 +669,9 @@
 
       final SessionReceiveMessage packet = new SessionReceiveMessage(id, message, ref.getDeliveryCount() + 1);
 
-      DelayedResult result = channel.replicatePacket(new SessionReplicateDeliveryMessage(id, message.getMessageID()));
+      DelayedResult result = channel.replicatePacket(new SessionReplicateDeliveryMessage(id,
+                                                                                         message.getMessageID(),
+                                                                                         message.getDestination()));
 
       if (result == null)
       {
@@ -590,9 +695,29 @@
    // Inner classes
    // ------------------------------------------------------------------------
 
+   final Runnable resumeLargeMessageRunnable = new Runnable()
+   {
+      public void run()
+      {
+         lock.lock();
+         try
+         {
+            if (largeMessageDeliverer == null || largeMessageDeliverer.deliver())
+            {
+               // prompt Delivery only if chunk was finished
+               session.promptDelivery(messageQueue);
+            }
+         }
+         finally
+         {
+            lock.unlock();
+         }
+      }
+   };
+
    /** Internal encapsulation of the logic on sending LargeMessages.
     *  This Inner class was created to avoid a bunch of loose properties about the current LargeMessage being sent*/
-   private class LargeMessageSender
+   private class LargeMessageDeliverer
    {
       private final long sizePendingLargeMessage;
 
@@ -606,9 +731,7 @@
       /** The current position on the message being processed */
       private volatile long positionPendingLargeMessage;
 
-      private volatile SessionReceiveContinuationMessage readAheadChunk;
-
-      public LargeMessageSender(final LargeServerMessage message, final MessageReference ref)
+      public LargeMessageDeliverer(final LargeServerMessage message, final MessageReference ref)
       {
          pendingLargeMessage = message;
 
@@ -617,7 +740,7 @@
          this.ref = ref;
       }
 
-      public boolean sendLargeMessage()
+      public boolean deliver()
       {
          lock.lock();
 
@@ -633,8 +756,23 @@
                return false;
             }
 
+            int creditsUsed;
+
+            if (availableCredits != null)
+            {
+               creditsUsed = preCalculateFlowControl();
+            }
+            else
+            {
+               creditsUsed = 0;
+            }
+
             if (!sentFirstMessage)
             {
+               if (trace)
+               {
+                  trace("deliverLargeMessage:: sending initialMessage, backup = " + messageQueue.isBackup());
+               }
                sentFirstMessage = true;
 
                MessagingBuffer headerBuffer = new ByteBufferWrapper(ByteBuffer.allocate(pendingLargeMessage.getPropertiesEncodeSize()));
@@ -649,34 +787,37 @@
 
                if (availableCredits != null)
                {
-                  // RequiredBufferSize on this case represents the right number of bytes sent
-                  availableCredits.addAndGet(-initialMessage.getRequiredBufferSize());
+                  if ((creditsUsed -= initialMessage.getRequiredBufferSize()) < 0)
+                  {
+                     log.warn("Credit logic is not working properly, too many credits were taken");
+                  }
+
+                  if (trace)
+                  {
+                     trace("deliverLargeMessage:: Initial send, taking out " + initialMessage.getRequiredBufferSize() +
+                           " credits, current = " +
+                           creditsUsed +
+                           " isBackup = " +
+                           messageQueue.isBackup());
+
+                  }
                }
             }
-
-            if (readAheadChunk != null)
+            else
             {
-               int chunkLen = readAheadChunk.getBody().length;
-
-               positionPendingLargeMessage += chunkLen;
-
-               if (availableCredits != null)
+               if (trace)
                {
-                  availableCredits.addAndGet(-readAheadChunk.getRequiredBufferSize());
+                  trace("deliverLargeMessage: Resuming deliverLargeMessage, currentPosition = " + positionPendingLargeMessage);
                }
-
-               channel.send(readAheadChunk);
-
-               readAheadChunk = null;
             }
 
             while (positionPendingLargeMessage < sizePendingLargeMessage)
             {
-               if (availableCredits != null && availableCredits.get() <= 0)
+               if (creditsUsed <= 0)
                {
-                  if (readAheadChunk == null)
+                  if (trace)
                   {
-                     readAheadChunk = createChunkSend();
+                     trace("deliverLargeMessage: Leaving loop of send LargeMessage because of credits, backup = " + messageQueue.isBackup());
                   }
                   return false;
                }
@@ -687,18 +828,42 @@
 
                if (availableCredits != null)
                {
-                  availableCredits.addAndGet(-chunk.getRequiredBufferSize());
+                  if ((creditsUsed -= chunk.getRequiredBufferSize()) < 0)
+                  {
+                     log.warn("Flowcontrol logic is not working properly, too many credits were taken");
+                  }
                }
 
+               if (trace)
+               {
+                  trace("deliverLargeMessage: Sending " + chunk.getRequiredBufferSize() +
+                        " availableCredits now is " +
+                        availableCredits +
+                        " isBackup = " +
+                        messageQueue.isBackup());
+               }
+
                channel.send(chunk);
 
                positionPendingLargeMessage += chunkLen;
             }
 
+            if (creditsUsed != 0)
+            {
+               log.warn("Flowcontrol logic is not working properly... creidts = " + creditsUsed);
+            }
+
+            if (trace)
+            {
+               trace("Finished deliverLargeMessage isBackup = " + messageQueue.isBackup());
+            }
+
             pendingLargeMessage.releaseResources();
 
-            largeMessageSender = null;
+            largeMessageDeliverer = null;
 
+            pendingLargeMessagesCounter.decrementAndGet();
+
             return true;
          }
          finally
@@ -707,6 +872,39 @@
          }
       }
 
+      /**
+       * Credits flow control are calculated in advance.
+       * @return
+       */
+      private int preCalculateFlowControl()
+      {
+         for (;;)
+         {
+            final int currentCredit;
+            int creditsUsed = 0;
+            currentCredit = availableCredits.get();
+
+            if (!sentFirstMessage)
+            {
+               creditsUsed = SessionReceiveMessage.SESSION_RECEIVE_MESSAGE_LARGE_MESSAGE_SIZE + pendingLargeMessage.getPropertiesEncodeSize();
+            }
+
+            long chunkLen = 0;
+            for (long i = positionPendingLargeMessage; creditsUsed < currentCredit && i < sizePendingLargeMessage; i += chunkLen)
+            {
+               chunkLen = (int)Math.min(sizePendingLargeMessage - i, minLargeMessageSize);
+               creditsUsed += chunkLen + SessionReceiveContinuationMessage.SESSION_RECEIVE_CONTINUATION_BASE_SIZE;
+            }
+
+            // The calculation of credits and taking credits out has to be taken atomically.
+            // Since we are not sending anything to the client during this calculation, this is unlikely to happen
+            if (availableCredits.compareAndSet(currentCredit, currentCredit - creditsUsed))
+            {
+               return creditsUsed;
+            }
+         }
+      }
+
       private SessionReceiveContinuationMessage createChunkSend()
       {
          SessionReceiveContinuationMessage chunk;

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/ServerMessageImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/ServerMessageImpl.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/ServerMessageImpl.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -140,7 +140,12 @@
    {
       return refCount.get();
    }
-   
+
+   public boolean isLargeMessage()
+   {
+      return false;
+   }
+
    public int getMemoryEstimate()
    {
       if (memoryEstimate == -1)

Modified: trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/src/main/org/jboss/messaging/core/server/impl/ServerSessionImpl.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -112,6 +112,11 @@
 
    private static final boolean trace = log.isTraceEnabled();
 
+   private static void trace(String message)
+   {
+      log.trace(message);
+   }
+
    // Static -------------------------------------------------------------------------------
 
    // Attributes ----------------------------------------------------------------------------
@@ -166,8 +171,16 @@
 
    private final QueueFactory queueFactory;
 
-   private volatile LargeServerMessage largeMessage;
+   // The current currentLargeMessage being processed
+   // In case of replication, currentLargeMessage should only be accessed within the replication callbacks 
+   private volatile LargeServerMessage currentLargeMessage;
 
+   // The current destination used for sending LargeMessages
+   /**
+    * We can't lookup for the address on the currentLargeMessage, as that is changed as the replication packets are coming backup from the backup node
+    */
+   private volatile SimpleString largeMessageCurrentAddress;
+
    // Constructors ---------------------------------------------------------------------------------
 
    public ServerSessionImpl(final String name,
@@ -298,11 +311,11 @@
 
       server.removeSession(name);
 
-      if (largeMessage != null)
+      if (currentLargeMessage != null)
       {
          try
          {
-            largeMessage.deleteFile();
+            currentLargeMessage.deleteFile();
          }
          catch (Throwable error)
          {
@@ -357,6 +370,12 @@
 
       if (result == null)
       {
+         if (trace)
+         {
+            trace("(NoReplication) CreateQueue address =  " + packet.getAddress() +
+                  " queueName = " +
+                  packet.getQueueName());
+         }
          doHandleCreateQueue(packet);
       }
       else
@@ -366,6 +385,12 @@
          {
             public void run()
             {
+               if (trace)
+               {
+                  trace("(Replication) CreateQueue address =  " + packet.getAddress() +
+                        " queueName = " +
+                        packet.getQueueName());
+               }
                doHandleCreateQueue(packet);
 
                lock.unlock();
@@ -394,6 +419,10 @@
 
       if (result == null)
       {
+         if (trace)
+         {
+            trace("(NoReplication) DeleteQueue queueName = " + packet.getQueueName());
+         }
          doHandleDeleteQueue(packet);
       }
       else
@@ -403,6 +432,10 @@
          {
             public void run()
             {
+               if (trace)
+               {
+                  trace("(Replication) DeleteQueue queueName = " + packet.getQueueName());
+               }
                doHandleDeleteQueue(packet);
 
                lock.unlock();
@@ -997,19 +1030,16 @@
    {
       DelayedResult result = channel.replicatePacket(packet);
 
-      try
-      {
-         // Note we don't wait for response before handling this
-
-         consumers.get(packet.getConsumerID()).receiveCredits(packet.getCredits());
-      }
-      catch (Exception e)
-      {
-         log.error("Failed to receive credits", e);
-      }
-
       if (result == null)
       {
+         try
+         {
+            consumers.get(packet.getConsumerID()).receiveCredits(packet.getCredits());
+         }
+         catch (Exception e)
+         {
+            log.error("Failed to receive credits", e);
+         }
          channel.confirm(packet);
       }
       else
@@ -1018,6 +1048,14 @@
          {
             public void run()
             {
+               try
+               {
+                  consumers.get(packet.getConsumerID()).receiveCredits(packet.getCredits());
+               }
+               catch (Exception e)
+               {
+                  log.error("Failed to receive credits", e);
+               }
                channel.confirm(packet);
             }
          });
@@ -1034,6 +1072,23 @@
          packet.setMessageID(id);
       }
 
+      // need to create the LargeMessage before continue
+      final LargeServerMessage msg = doCreateLargeMessage(packet);
+
+      if (msg == null)
+      {
+         // packet logged an error, and played with channel.returns... and nothing needs to be done now
+         return;
+      }
+
+      largeMessageCurrentAddress = msg.getDestination();
+
+      if (channel.getReplicatingChannel() != null)
+      {
+         msg.putBooleanProperty(new SimpleString("clustered"), true);
+      }
+
+      // Note: We don't need to use address lock until the last packet
       DelayedResult result = channel.replicatePacket(packet);
 
       // With a send we must make sure it is replicated to backup before being processed on live
@@ -1041,6 +1096,17 @@
 
       if (result == null)
       {
+         if (trace)
+         {
+            trace("(withoutReplication) SendLargeMessage, id=" + msg.getMessageID());
+         }
+
+         if (currentLargeMessage != null)
+         {
+            log.warn("Replacing incomplete LargeMessage with ID=" + currentLargeMessage.getMessageID());
+         }
+
+         currentLargeMessage = msg;
          doSendLargeMessage(packet);
       }
       else
@@ -1049,9 +1115,21 @@
          {
             public void run()
             {
+               if (trace)
+               {
+                  trace("(Replication) SendLargeMessage, id=" + msg.getMessageID());
+               }
+
+               if (currentLargeMessage != null)
+               {
+                  log.warn("Replacing incomplete LargeMessage with ID=" + currentLargeMessage.getMessageID());
+               }
+
+               currentLargeMessage = msg;
                doSendLargeMessage(packet);
             }
          });
+
       }
 
    }
@@ -1114,13 +1192,31 @@
 
    public void handleSendContinuations(final SessionSendContinuationMessage packet)
    {
+      final SendLock lock;
+
+      // We only use the addressLock at the last packet
+      if (channel.getReplicatingChannel() != null && !packet.isContinues())
+      {
+         lock = postOffice.getAddressLock(largeMessageCurrentAddress);
+
+         lock.beforeSend();
+      }
+      else
+      {
+         lock = null;
+      }
+
       DelayedResult result = channel.replicatePacket(packet);
 
-      // With a send we must make sure it is replicated to backup before being processed on live
-      // or can end up with delivery being processed on backup before original send
-
       if (result == null)
       {
+         if (trace)
+         {
+            if (!packet.isContinues())
+            {
+               trace("(NoReplication) Sending LasChunk MessageID = " + currentLargeMessage.getMessageID());
+            }
+         }
          doSendContinuations(packet);
       }
       else
@@ -1129,7 +1225,15 @@
          {
             public void run()
             {
+               if (trace && !packet.isContinues())
+               {
+                  trace("(Replication) Sending LasChunk MessageID = " + currentLargeMessage.getMessageID());
+               }
                doSendContinuations(packet);
+               if (lock != null)
+               {
+                  lock.afterSend();
+               }
             }
          });
       }
@@ -1146,7 +1250,7 @@
 
       try
       {
-         consumer.deliverReplicated(packet.getMessageID());
+         consumer.deliverReplicated(packet.getAddress(), packet.getMessageID());
       }
       catch (Exception e)
       {
@@ -1304,9 +1408,11 @@
                                                           filter,
                                                           started,
                                                           browseOnly,
-                                                          storageManager,                                                 
+                                                          storageManager,
+                                                          postOffice.getPagingManager(),
                                                           channel,
-                                                          preAcknowledge);
+                                                          preAcknowledge,
+                                                          executor);
 
          consumers.put(consumer.getID(), consumer);
 
@@ -1374,8 +1480,8 @@
          if (durable)
          {
             QueueBinding queueBinding = (QueueBinding)binding;
-            
-            storageManager.addQueueBinding(queueBinding);                        
+
+            storageManager.addQueueBinding(queueBinding);
          }
 
          postOffice.addBinding(binding);
@@ -2359,14 +2465,48 @@
       started = s;
    }
 
+   /**
+    * We need to create the LargeMessage before replicating the packe, or else we won't know how to extract the destination,
+    * which is stored on the header
+    * @param packet
+    * @throws Exception
+    */
+   private LargeServerMessage doCreateLargeMessage(final SessionSendMessage packet)
+   {
+      try
+      {
+         return createLargeMessageStorage(packet.getMessageID(), packet.getLargeMessageHeader());
+      }
+      catch (Exception e)
+      {
+         log.error("Failed to create large message", e);
+         Packet response = null;
+         if (packet.isRequiresResponse())
+         {
+            if (e instanceof MessagingException)
+            {
+               response = new MessagingExceptionMessage((MessagingException)e);
+            }
+            else
+            {
+               response = new MessagingExceptionMessage(new MessagingException(MessagingException.INTERNAL_ERROR));
+            }
+         }
+         channel.confirm(packet);
+         if (response != null)
+         {
+            channel.send(response);
+         }
+         return null;
+      }
+   }
+
    private void doSendLargeMessage(final SessionSendMessage packet)
    {
       Packet response = null;
 
       try
       {
-         largeMessage = createLargeMessageStorage(packet.getMessageID(), packet.getLargeMessageHeader());
-
          if (packet.isRequiresResponse())
          {
             response = new NullResponseMessage();
@@ -2456,18 +2596,18 @@
       try
       {
 
-         if (largeMessage == null)
+         if (currentLargeMessage == null)
          {
             throw new MessagingException(MessagingException.ILLEGAL_STATE, "large-message not initialized on server");
          }
 
-         largeMessage.addBytes(packet.getBody());
+         currentLargeMessage.addBytes(packet.getBody());
 
          if (!packet.isContinues())
          {
-            final LargeServerMessage message = largeMessage;
+            final LargeServerMessage message = currentLargeMessage;
 
-            largeMessage = null;
+            currentLargeMessage = null;
 
             message.complete();
 
@@ -2526,7 +2666,7 @@
 
       largeMessage.decodeProperties(headerBuffer);
 
-      // client didn send the ID originally
+      // client didn't send the ID originally
       largeMessage.setMessageID(messageID);
 
       return largeMessage;

Modified: trunk/tests/src/org/jboss/messaging/tests/integration/chunkmessage/MessageChunkTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/chunkmessage/MessageChunkTest.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/chunkmessage/MessageChunkTest.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -524,6 +524,85 @@
 
    }
 
+   public void testSimpleRollback() throws Exception
+   {
+      // there are two bindings.. one is ACKed, the other is not, the server is restarted
+      // The other binding is acked... The file must be deleted
+
+      clearData();
+
+      try
+      {
+
+         messagingService = createService(true);
+
+         messagingService.start();
+
+         ClientSessionFactory sf = createInVMFactory();
+
+         ClientSession session = sf.createSession(false, false, false);
+
+         session.createQueue(ADDRESS, ADDRESS, null, true, false);
+
+         int numberOfIntegers = 100;
+
+         Message clientFile = createLargeClientMessage(session, numberOfIntegers);
+
+         session.start();
+
+         log.info ("Session started");
+         
+         ClientProducer producer = session.createProducer(ADDRESS);
+
+         ClientConsumer consumer = session.createConsumer(ADDRESS);
+         
+         for (int n = 0; n < 10; n++)
+         {
+            producer.send(clientFile);
+
+            assertNull(consumer.receiveImmediate());
+
+            session.commit();
+
+            for (int i = 0; i < 2; i++)
+            {
+
+               ClientMessage clientMessage = consumer.receive(5000);
+
+               assertNotNull(clientMessage);
+
+               assertEquals(numberOfIntegers * 4, clientMessage.getBody().limit());
+
+               clientMessage.acknowledge();
+
+               if (i == 0)
+               {
+                  session.rollback();
+               }
+               else
+               {
+                  session.commit();
+               }
+            }
+         }
+
+         session.close();
+
+         validateNoFilesOnLargeDir();
+      }
+      finally
+      {
+         try
+         {
+            messagingService.stop();
+         }
+         catch (Throwable ignored)
+         {
+         }
+      }
+
+   }
+
    // Package protected ---------------------------------------------
 
    // Protected -----------------------------------------------------

Modified: trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/FailoverTestBase.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/FailoverTestBase.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/FailoverTestBase.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -56,9 +56,9 @@
 
    protected final Map<String, Object> backupParams = new HashMap<String, Object>();
 
-   private MessagingService liveService;
+   protected MessagingService liveService;
 
-   private MessagingService backupService;
+   protected MessagingService backupService;
 
    // Static --------------------------------------------------------
 
@@ -73,9 +73,20 @@
                                                                      backupParams));
    }
 
-   protected void setUpFileBased() throws Exception
+   protected ClientSessionFactory createBackupFactory()
    {
+      return new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
+                                                                     backupParams));
+   }
 
+   protected void setUpFileBased(final long maxGlobalSize) throws Exception
+   {
+      setUpFileBased(maxGlobalSize, 20 * 1024);
+   }
+
+   protected void setUpFileBased(final long maxGlobalSize, final long pageSize) throws Exception
+   {
+
       deleteDirectory(new File(getTestDir()));
 
       Configuration backupConf = new ConfigurationImpl();
@@ -86,8 +97,8 @@
       backupConf.setPagingDirectory(getPageDir(getTestDir() + "/backup"));
       backupConf.setJournalFileSize(100 * 1024);
 
-      backupConf.setPagingMaxGlobalSizeBytes(100 * 1024 * 1024);
-      backupConf.setPagingDefaultSize(10 * 1024);
+      backupConf.setPagingMaxGlobalSizeBytes(maxGlobalSize);
+      backupConf.setPagingDefaultSize(pageSize);
 
       backupConf.setSecurityEnabled(false);
       backupParams.put(TransportConstants.SERVER_ID_PROP_NAME, 1);
@@ -108,8 +119,8 @@
       liveConf.setBindingsDirectory(getBindingsDir(getTestDir() + "/live"));
       liveConf.setPagingDirectory(getPageDir(getTestDir() + "/live"));
 
-      liveConf.setPagingMaxGlobalSizeBytes(100 * 1024 * 1024);
-      liveConf.setPagingDefaultSize(10 * 1024);
+      liveConf.setPagingMaxGlobalSizeBytes(maxGlobalSize);
+      liveConf.setPagingDefaultSize(pageSize);
       liveConf.setJournalFileSize(100 * 1024);
 
       liveConf.setSecurityEnabled(false);
@@ -135,22 +146,37 @@
 
    // Protected -----------------------------------------------------
 
-   @Override
    protected void tearDown() throws Exception
    {
-      assertEquals(0, backupService.getServer().getRemotingService().getConnections().size());
-
-      backupService.stop();
-
-      assertEquals(0, liveService.getServer().getRemotingService().getConnections().size());
-
-      liveService.stop();
-
-      assertEquals(0, InVMRegistry.instance.size());
+      stopServers();
       
       super.tearDown();
    }
+   
+   protected void stopServers() throws Exception
+   {
+      if (backupService != null && backupService.isStarted())
+      {
+         assertEquals(0, backupService.getServer().getRemotingService().getConnections().size());
+   
+         backupService.stop();
+         
+         backupService = null;
+      }
+      
+      if (liveService != null && liveService.isStarted())
+      {
+         assertEquals(0, liveService.getServer().getRemotingService().getConnections().size());
+   
+         liveService.stop();
+         
+         liveService = null;
+   
+      }
 
+      assertEquals(0, InVMRegistry.instance.size());
+   }
+
    // Private -------------------------------------------------------
 
    // Inner classes -------------------------------------------------

Modified: trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/JustReplicationTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/JustReplicationTest.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/JustReplicationTest.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -68,8 +68,7 @@
       factory.setBlockOnNonPersistentSend(true);
       factory.setBlockOnPersistentSend(true);
       
-      // Enable this and the test will fail
-      //factory.setMinLargeMessageSize(10 * 1024);
+      factory.setMinLargeMessageSize(10 * 1024);
 
       ClientSession session = factory.createSession(null, null, false, true, true, false, 0);
 
@@ -145,7 +144,7 @@
    @Override
    protected void setUp() throws Exception
    {
-      setUpFileBased();
+      setUpFileBased(100*1024*1024);
    }
 
    protected void tearDown() throws Exception

Added: trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/LargeMessageMuliThreadFailoverTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/LargeMessageMuliThreadFailoverTest.java	                        (rev 0)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/LargeMessageMuliThreadFailoverTest.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -0,0 +1,154 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.tests.integration.cluster.failover;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.jboss.messaging.core.client.ClientMessage;
+import org.jboss.messaging.core.client.impl.ClientSessionFactoryInternal;
+import org.jboss.messaging.core.config.Configuration;
+import org.jboss.messaging.core.config.TransportConfiguration;
+import org.jboss.messaging.core.config.impl.ConfigurationImpl;
+import org.jboss.messaging.core.remoting.impl.ByteBufferWrapper;
+import org.jboss.messaging.core.remoting.impl.invm.InVMAcceptorFactory;
+import org.jboss.messaging.core.remoting.impl.invm.TransportConstants;
+import org.jboss.messaging.core.server.impl.MessagingServiceImpl;
+
+/**
+ * A LargeMessageMuliThreadFailoverTest
+ *
+ * @author <a href="mailto:clebert.suconic at jboss.org">Clebert Suconic</a>
+ * 
+ * Created Jan 18, 2009 4:52:09 PM
+ *
+ *
+ */
+public class LargeMessageMuliThreadFailoverTest extends MultiThreadRandomFailoverTestBase
+{
+
+   // Constants -----------------------------------------------------
+
+   // Attributes ----------------------------------------------------
+
+   // Static --------------------------------------------------------
+
+   // Constructors --------------------------------------------------
+
+   // Public --------------------------------------------------------
+
+   // Package protected ---------------------------------------------
+
+   // Protected -----------------------------------------------------
+   
+   protected ClientSessionFactoryInternal createSessionFactory()
+   {
+      ClientSessionFactoryInternal sf = super.createSessionFactory();
+      
+      sf.setMinLargeMessageSize(200);
+      
+      return sf;
+
+   }
+
+
+   @Override
+   protected void start() throws Exception
+   {
+
+      deleteDirectory(new File(getTestDir()));
+
+      Configuration backupConf = new ConfigurationImpl();
+
+      backupConf.setJournalDirectory(getJournalDir(getTestDir() + "/backup"));
+      backupConf.setLargeMessagesDirectory(getLargeMessagesDir(getTestDir() + "/backup"));
+      backupConf.setBindingsDirectory(getBindingsDir(getTestDir() + "/backup"));
+      backupConf.setPagingDirectory(getPageDir(getTestDir() + "/backup"));
+      backupConf.setJournalFileSize(100 * 1024);
+
+      backupConf.setSecurityEnabled(false);
+      backupParams.put(TransportConstants.SERVER_ID_PROP_NAME, 1);
+
+      backupConf.getAcceptorConfigurations()
+                .add(new TransportConfiguration(InVMAcceptorFactory.class.getCanonicalName(), backupParams));
+      backupConf.setBackup(true);
+
+      backupService = MessagingServiceImpl.newMessagingService(backupConf);
+      backupService.start();
+
+      Configuration liveConf = new ConfigurationImpl();
+
+      liveConf.setJournalDirectory(getJournalDir(getTestDir() + "/live"));
+      liveConf.setLargeMessagesDirectory(getLargeMessagesDir(getTestDir() + "/live"));
+      liveConf.setBindingsDirectory(getBindingsDir(getTestDir() + "/live"));
+      liveConf.setPagingDirectory(getPageDir(getTestDir() + "/live"));
+
+      liveConf.setJournalFileSize(100 * 1024);
+
+      liveConf.setSecurityEnabled(false);
+      liveConf.getAcceptorConfigurations()
+              .add(new TransportConfiguration(InVMAcceptorFactory.class.getCanonicalName()));
+
+      Map<String, TransportConfiguration> connectors = new HashMap<String, TransportConfiguration>();
+
+      TransportConfiguration backupTC = new TransportConfiguration(INVM_CONNECTOR_FACTORY,
+                                                                   backupParams,
+                                                                   "backup-connector");
+      connectors.put(backupTC.getName(), backupTC);
+      liveConf.setConnectorConfigurations(connectors);
+      liveConf.setBackupConnectorName(backupTC.getName());
+      liveService = MessagingServiceImpl.newMessagingService(liveConf);
+
+      liveService.start();
+
+   }
+   
+   protected int getNumIterations()
+   {
+      return 10;
+   }
+
+
+   @Override
+   protected void setBody(final ClientMessage message) throws Exception
+   {
+      message.setBody(new ByteBufferWrapper(ByteBuffer.allocate(500)));
+
+   }
+
+   /* (non-Javadoc)
+    * @see org.jboss.messaging.tests.integration.cluster.failover.MultiThreadRandomFailoverTestBase#checkSize(org.jboss.messaging.core.client.ClientMessage)
+    */
+   @Override
+   protected boolean checkSize(ClientMessage message)
+   {
+      return 500 ==  message.getBodySize();
+   }
+
+   // Private -------------------------------------------------------
+
+   // Inner classes -------------------------------------------------
+
+}

Modified: trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadRandomFailoverTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadRandomFailoverTest.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadRandomFailoverTest.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -12,1329 +12,27 @@
 
 package org.jboss.messaging.tests.integration.cluster.failover;
 
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
 
-import junit.framework.TestCase;
-
-import org.jboss.messaging.core.client.ClientConsumer;
 import org.jboss.messaging.core.client.ClientMessage;
-import org.jboss.messaging.core.client.ClientProducer;
-import org.jboss.messaging.core.client.ClientSession;
-import org.jboss.messaging.core.client.ClientSessionFactory;
-import org.jboss.messaging.core.client.MessageHandler;
-import org.jboss.messaging.core.client.impl.ClientSessionFactoryImpl;
-import org.jboss.messaging.core.client.impl.ClientSessionFactoryInternal;
-import org.jboss.messaging.core.client.impl.ClientSessionImpl;
 import org.jboss.messaging.core.config.Configuration;
 import org.jboss.messaging.core.config.TransportConfiguration;
 import org.jboss.messaging.core.config.impl.ConfigurationImpl;
-import org.jboss.messaging.core.exception.MessagingException;
-import org.jboss.messaging.core.logging.Logger;
-import org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl;
-import org.jboss.messaging.core.remoting.impl.invm.InVMConnector;
-import org.jboss.messaging.core.remoting.impl.invm.InVMRegistry;
 import org.jboss.messaging.core.remoting.impl.invm.TransportConstants;
-import org.jboss.messaging.core.server.MessagingService;
 import org.jboss.messaging.core.server.impl.MessagingServiceImpl;
-import org.jboss.messaging.jms.client.JBossTextMessage;
-import org.jboss.messaging.util.SimpleString;
 
 /**
  * A MultiThreadRandomFailoverTest
  * 
  * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
+ * @author <a href="mailto:clebert.suconic at jboss.org">Clebert Suconic</a>
  */
-public class MultiThreadRandomFailoverTest extends TestCase
+public class MultiThreadRandomFailoverTest extends MultiThreadRandomFailoverTestBase
 {
-   private static final Logger log = Logger.getLogger(MultiThreadRandomFailoverTest.class);
-
-   // Constants -----------------------------------------------------
-
-   private static final int RECEIVE_TIMEOUT = 30000;
-
-   private static final int NUM_THREADS = 10;
-
-   // Attributes ----------------------------------------------------
-   private static final SimpleString ADDRESS = new SimpleString("FailoverTestAddress");
-
-   private MessagingService liveService;
-
-   private MessagingService backupService;
-
-   private final Map<String, Object> backupParams = new HashMap<String, Object>();
-
-   private Timer timer;
-
-   // Static --------------------------------------------------------
-
-   // Constructors --------------------------------------------------
-
-   // Public --------------------------------------------------------
-
-   public void testA() throws Exception
+   @Override
+   protected void start() throws Exception
    {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestA(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testB() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestB(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testC() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestC(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testD() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestD(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testE() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestE(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testF() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestF(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testG() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestG(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testH() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestH(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testI() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestI(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testJ() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestJ(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testK() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestK(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   public void testL() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestL(sf);
-         }
-      }, NUM_THREADS);
-   }
-
-   // public void testM() throws Exception
-   // {
-   // runTestMultipleThreads(new RunnableT()
-   // {
-   // public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-   // {
-   // doTestM(sf, threadNum);
-   // }
-   // }, NUM_THREADS);
-   // }
-
-   public void testN() throws Exception
-   {
-      runTestMultipleThreads(new RunnableT()
-      {
-         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
-         {
-            doTestN(sf, threadNum);
-         }
-      }, NUM_THREADS);
-   }
-
-   // Package protected ---------------------------------------------
-
-   // Protected -----------------------------------------------------
-
-   protected void doTestA(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      long start = System.currentTimeMillis();
-
-      ClientSession s = sf.createSession(false, false, false);
-
-      final int numMessages = 100;
-
-      final int numSessions = 10;
-
-      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
-      Set<ClientSession> sessions = new HashSet<ClientSession>();
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         ClientSession sessConsume = sf.createSession(false, true, true);
-
-         sessConsume.start();
-
-         sessConsume.createQueue(ADDRESS, subName, null, false, false);
-
-         ClientConsumer consumer = sessConsume.createConsumer(subName);
-
-         consumers.add(consumer);
-
-         sessions.add(sessConsume);
-      }
-
-      ClientSession sessSend = sf.createSession(false, true, true);
-
-      ClientProducer producer = sessSend.createProducer(ADDRESS);
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      Set<MyHandler> handlers = new HashSet<MyHandler>();
-
-      for (ClientConsumer consumer : consumers)
-      {
-         MyHandler handler = new MyHandler(threadNum, numMessages);
-
-         consumer.setMessageHandler(handler);
-
-         handlers.add(handler);
-      }
-
-      for (MyHandler handler : handlers)
-      {
-         boolean ok = handler.latch.await(5000, TimeUnit.MILLISECONDS);
-
-         if (!ok)
-         {
-            throw new Exception("Timed out waiting for messages on handler " + System.identityHashCode(handler) +
-                                " threadnum " +
-                                threadNum);
-         }
-
-         if (handler.failure != null)
-         {
-            throw new Exception("Handler failed: " + handler.failure);
-         }
-      }
-
-      sessSend.close();
-      for (ClientSession session : sessions)
-      {
-         session.close();
-      }
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         s.deleteQueue(subName);
-      }
-
-      s.close();
-
-      long end = System.currentTimeMillis();
-
-      log.info("duration " + (end - start));
-   }
-
-   protected void doTestB(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      long start = System.currentTimeMillis();
-
-      ClientSession s = sf.createSession(false, false, false);
-
-      final int numMessages = 100;
-
-      final int numSessions = 10;
-
-      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
-      Set<ClientSession> sessions = new HashSet<ClientSession>();
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         ClientSession sessConsume = sf.createSession(false, true, true);
-
-         sessConsume.createQueue(ADDRESS, subName, null, false, false);
-
-         ClientConsumer consumer = sessConsume.createConsumer(subName);
-
-         consumers.add(consumer);
-
-         sessions.add(sessConsume);
-      }
-
-      ClientSession sessSend = sf.createSession(false, true, true);
-
-      ClientProducer producer = sessSend.createProducer(ADDRESS);
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      for (ClientSession session : sessions)
-      {
-         session.start();
-      }
-
-      Set<MyHandler> handlers = new HashSet<MyHandler>();
-
-      for (ClientConsumer consumer : consumers)
-      {
-         MyHandler handler = new MyHandler(threadNum, numMessages);
-
-         consumer.setMessageHandler(handler);
-
-         handlers.add(handler);
-      }
-
-      for (MyHandler handler : handlers)
-      {
-         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
-
-         if (!ok)
-         {
-            throw new Exception("Timed out waiting for messages on handler " + System.identityHashCode(handler) +
-                                " threadnum " +
-                                threadNum);
-         }
-
-         if (handler.failure != null)
-         {
-            throw new Exception("Handler failed: " + handler.failure);
-         }
-      }
-
-      sessSend.close();
-
-      for (ClientSession session : sessions)
-      {
-         session.close();
-      }
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         s.deleteQueue(subName);
-      }
-
-      s.close();
-
-      long end = System.currentTimeMillis();
-
-      log.info("duration " + (end - start));
-
-   }
-
-   protected void doTestC(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      long start = System.currentTimeMillis();
-
-      ClientSession s = sf.createSession(false, false, false);
-
-      final int numMessages = 100;
-
-      final int numSessions = 10;
-
-      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
-      Set<ClientSession> sessions = new HashSet<ClientSession>();
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         ClientSession sessConsume = sf.createSession(false, false, false);
-
-         sessConsume.start();
-
-         sessConsume.createQueue(ADDRESS, subName, null, false, false);
-
-         ClientConsumer consumer = sessConsume.createConsumer(subName);
-
-         consumers.add(consumer);
-
-         sessions.add(sessConsume);
-      }
-
-      ClientSession sessSend = sf.createSession(false, false, false);
-
-      ClientProducer producer = sessSend.createProducer(ADDRESS);
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      sessSend.rollback();
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      sessSend.commit();
-
-      Set<MyHandler> handlers = new HashSet<MyHandler>();
-
-      for (ClientConsumer consumer : consumers)
-      {
-         MyHandler handler = new MyHandler(threadNum, numMessages);
-
-         consumer.setMessageHandler(handler);
-
-         handlers.add(handler);
-      }
-
-      for (MyHandler handler : handlers)
-      {
-         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
-
-         if (!ok)
-         {
-            throw new Exception("Timed out waiting for messages on handler " + System.identityHashCode(handler) +
-                                " threadnum " +
-                                threadNum);
-         }
-
-         if (handler.failure != null)
-         {
-            throw new Exception("Handler failed: " + handler.failure);
-         }
-      }
-
-      for (ClientSession session : sessions)
-      {
-         session.rollback();
-      }
-
-      for (MyHandler handler : handlers)
-      {
-         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
-
-         assertTrue(ok);
-      }
-
-      for (ClientSession session : sessions)
-      {
-         session.commit();
-      }
-
-      sessSend.close();
-      for (ClientSession session : sessions)
-      {
-         session.close();
-      }
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         s.deleteQueue(subName);
-      }
-
-      s.close();
-
-      long end = System.currentTimeMillis();
-
-      log.info("duration " + (end - start));
-   }
-
-   protected void doTestD(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      long start = System.currentTimeMillis();
-
-      ClientSession s = sf.createSession(false, false, false);
-
-      final int numMessages = 100;
-
-      final int numSessions = 10;
-
-      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
-      Set<ClientSession> sessions = new HashSet<ClientSession>();
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + " sub" + i);
-
-         ClientSession sessConsume = sf.createSession(false, false, false);
-
-         sessConsume.createQueue(ADDRESS, subName, null, false, false);
-
-         ClientConsumer consumer = sessConsume.createConsumer(subName);
-
-         consumers.add(consumer);
-
-         sessions.add(sessConsume);
-      }
-
-      ClientSession sessSend = sf.createSession(false, false, false);
-
-      ClientProducer producer = sessSend.createProducer(ADDRESS);
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      sessSend.rollback();
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      sessSend.commit();
-
-      for (ClientSession session : sessions)
-      {
-         session.start();
-      }
-
-      Set<MyHandler> handlers = new HashSet<MyHandler>();
-
-      for (ClientConsumer consumer : consumers)
-      {
-         MyHandler handler = new MyHandler(threadNum, numMessages);
-
-         consumer.setMessageHandler(handler);
-
-         handlers.add(handler);
-      }
-
-      for (MyHandler handler : handlers)
-      {
-         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
-
-         if (!ok)
-         {
-            throw new Exception("Timed out waiting for messages on handler " + System.identityHashCode(handler) +
-                                " threadnum " +
-                                threadNum);
-         }
-
-         if (handler.failure != null)
-         {
-            throw new Exception("Handler failed: " + handler.failure);
-         }
-      }
-
-      handlers.clear();
-
-      // Set handlers to null
-      for (ClientConsumer consumer : consumers)
-      {
-         consumer.setMessageHandler(null);
-      }
-
-      for (ClientSession session : sessions)
-      {
-         session.rollback();
-      }
-
-      // New handlers
-      for (ClientConsumer consumer : consumers)
-      {
-         MyHandler handler = new MyHandler(threadNum, numMessages);
-
-         consumer.setMessageHandler(handler);
-
-         handlers.add(handler);
-      }
-
-      for (MyHandler handler : handlers)
-      {
-         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
-
-         if (!ok)
-         {
-            throw new Exception("Timed out waiting for messages on handler " + System.identityHashCode(handler) +
-                                " threadnum " +
-                                threadNum);
-         }
-
-         if (handler.failure != null)
-         {
-            throw new Exception("Handler failed on rollback: " + handler.failure);
-         }
-      }
-
-      for (ClientSession session : sessions)
-      {
-         session.commit();
-      }
-
-      sessSend.close();
-      for (ClientSession session : sessions)
-      {
-         session.close();
-      }
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + " sub" + i);
-
-         s.deleteQueue(subName);
-      }
-
-      s.close();
-
-      long end = System.currentTimeMillis();
-
-      log.info("duration " + (end - start));
-   }
-
-   // Now with synchronous receive()
-
-   protected void doTestE(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      long start = System.currentTimeMillis();
-
-      ClientSession s = sf.createSession(false, false, false);
-
-      final int numMessages = 100;
-
-      final int numSessions = 10;
-
-      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
-      Set<ClientSession> sessions = new HashSet<ClientSession>();
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         ClientSession sessConsume = sf.createSession(false, true, true);
-
-         sessConsume.start();
-
-         sessConsume.createQueue(ADDRESS, subName, null, false, false);
-
-         ClientConsumer consumer = sessConsume.createConsumer(subName);
-
-         consumers.add(consumer);
-
-         sessions.add(sessConsume);
-      }
-
-      ClientSession sessSend = sf.createSession(false, true, true);
-
-      ClientProducer producer = sessSend.createProducer(ADDRESS);
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      consumeMessages(consumers, numMessages, threadNum);
-
-      sessSend.close();
-      for (ClientSession session : sessions)
-      {
-         session.close();
-      }
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         s.deleteQueue(subName);
-      }
-
-      s.close();
-
-      long end = System.currentTimeMillis();
-
-      log.info("duration " + (end - start));
-   }
-
-   protected void doTestF(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      long start = System.currentTimeMillis();
-
-      ClientSession s = sf.createSession(false, false, false);
-
-      final int numMessages = 100;
-
-      final int numSessions = 10;
-
-      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
-      Set<ClientSession> sessions = new HashSet<ClientSession>();
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         ClientSession sessConsume = sf.createSession(false, true, true);
-
-         sessConsume.createQueue(ADDRESS, subName, null, false, false);
-
-         ClientConsumer consumer = sessConsume.createConsumer(subName);
-
-         consumers.add(consumer);
-
-         sessions.add(sessConsume);
-      }
-
-      ClientSession sessSend = sf.createSession(false, true, true);
-
-      ClientProducer producer = sessSend.createProducer(ADDRESS);
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      for (ClientSession session : sessions)
-      {
-         session.start();
-      }
-
-      consumeMessages(consumers, numMessages, threadNum);
-
-      sessSend.close();
-      for (ClientSession session : sessions)
-      {
-         session.close();
-      }
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         s.deleteQueue(subName);
-      }
-
-      s.close();
-
-      long end = System.currentTimeMillis();
-
-      log.info("duration " + (end - start));
-   }
-
-   protected void doTestG(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      long start = System.currentTimeMillis();
-
-      ClientSession s = sf.createSession(false, false, false);
-
-      final int numMessages = 100;
-
-      final int numSessions = 10;
-
-      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
-      Set<ClientSession> sessions = new HashSet<ClientSession>();
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         ClientSession sessConsume = sf.createSession(false, false, false);
-
-         sessConsume.start();
-
-         sessConsume.createQueue(ADDRESS, subName, null, false, false);
-
-         ClientConsumer consumer = sessConsume.createConsumer(subName);
-
-         consumers.add(consumer);
-
-         sessions.add(sessConsume);
-      }
-
-      ClientSession sessSend = sf.createSession(false, false, false);
-
-      ClientProducer producer = sessSend.createProducer(ADDRESS);
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      sessSend.rollback();
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      sessSend.commit();
-
-      consumeMessages(consumers, numMessages, threadNum);
-
-      for (ClientSession session : sessions)
-      {
-         session.rollback();
-      }
-
-      consumeMessages(consumers, numMessages, threadNum);
-
-      for (ClientSession session : sessions)
-      {
-         session.commit();
-      }
-
-      sessSend.close();
-      for (ClientSession session : sessions)
-      {
-         session.close();
-      }
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         s.deleteQueue(subName);
-      }
-
-      s.close();
-
-      long end = System.currentTimeMillis();
-
-      log.info("duration " + (end - start));
-   }
-
-   protected void doTestH(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      long start = System.currentTimeMillis();
-
-      ClientSession s = sf.createSession(false, false, false);
-
-      final int numMessages = 100;
-
-      final int numSessions = 10;
-
-      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
-      Set<ClientSession> sessions = new HashSet<ClientSession>();
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         ClientSession sessConsume = sf.createSession(false, false, false);
-
-         sessConsume.createQueue(ADDRESS, subName, null, false, false);
-
-         ClientConsumer consumer = sessConsume.createConsumer(subName);
-
-         consumers.add(consumer);
-
-         sessions.add(sessConsume);
-      }
-
-      ClientSession sessSend = sf.createSession(false, false, false);
-
-      ClientProducer producer = sessSend.createProducer(ADDRESS);
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      sessSend.rollback();
-
-      sendMessages(sessSend, producer, numMessages, threadNum);
-
-      sessSend.commit();
-
-      for (ClientSession session : sessions)
-      {
-         session.start();
-      }
-
-      consumeMessages(consumers, numMessages, threadNum);
-
-      for (ClientSession session : sessions)
-      {
-         session.rollback();
-      }
-
-      consumeMessages(consumers, numMessages, threadNum);
-
-      for (ClientSession session : sessions)
-      {
-         session.commit();
-      }
-
-      sessSend.close();
-      for (ClientSession session : sessions)
-      {
-         session.close();
-      }
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         SimpleString subName = new SimpleString(threadNum + "sub" + i);
-
-         s.deleteQueue(subName);
-      }
-
-      s.close();
-
-      long end = System.currentTimeMillis();
-
-      log.info("duration " + (end - start));
-   }
-
-   protected void doTestI(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      ClientSession sessCreate = sf.createSession(false, true, true);
-
-      sessCreate.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
-
-      ClientSession sess = sf.createSession(false, true, true);
-
-      sess.start();
-
-      ClientConsumer consumer = sess.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
-
-      ClientProducer producer = sess.createProducer(ADDRESS);
-
-      ClientMessage message = sess.createClientMessage(JBossTextMessage.TYPE,
-                                                       false,
-                                                       0,
-                                                       System.currentTimeMillis(),
-                                                       (byte)1);
-      message.getBody().flip();
-
-      producer.send(message);
-
-      ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
-
-      assertNotNull(message2);
-
-      message2.acknowledge();
-
-      sess.close();
-
-      sessCreate.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
-
-      sessCreate.close();
-   }
-
-   protected void doTestJ(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      ClientSession sessCreate = sf.createSession(false, true, true);
-
-      sessCreate.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
-
-      ClientSession sess = sf.createSession(false, true, true);
-
-      sess.start();
-
-      ClientConsumer consumer = sess.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
-
-      ClientProducer producer = sess.createProducer(ADDRESS);
-
-      ClientMessage message = sess.createClientMessage(JBossTextMessage.TYPE,
-                                                       false,
-                                                       0,
-                                                       System.currentTimeMillis(),
-                                                       (byte)1);
-      message.getBody().flip();
-
-      producer.send(message);
-
-      ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
-
-      assertNotNull(message2);
-
-      message2.acknowledge();
-
-      sess.close();
-
-      sessCreate.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
-
-      sessCreate.close();
-   }
-
-   protected void doTestK(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      ClientSession s = sf.createSession(false, false, false);
-
-      s.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
-
-      final int numConsumers = 100;
-
-      for (int i = 0; i < numConsumers; i++)
-      {
-         ClientConsumer consumer = s.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
-
-         consumer.close();
-      }
-
-      s.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
-
-      s.close();
-   }
-
-   protected void doTestL(final ClientSessionFactory sf) throws Exception
-   {
-      ClientSession s = sf.createSession(false, false, false);
-
-      final int numSessions = 100;
-
-      for (int i = 0; i < numSessions; i++)
-      {
-         ClientSession session = sf.createSession(false, false, false);
-
-         session.close();
-      }
-
-      s.close();
-   }
-
-   // Browsers
-   // FIXME - this test won't work until we use a proper iterator for browsing a queue.
-   // Making a copy of the queue for a browser consumer doesn't work well with replication since
-   // When replicating the create consumer (browser) to the backup, when executed on the backup the
-   // backup may have different messages in its queue since been added on different threads.
-   // So when replicating deliveries they may not be found.
-   // https://jira.jboss.org/jira/browse/JBMESSAGING-1433
-   // protected void doTestM(final ClientSessionFactory sf, final int threadNum) throws Exception
-   // {
-   // long start = System.currentTimeMillis();
-   //
-   // ClientSession sessSend = sf.createSession(false, true, true, false);
-   //      
-   // ClientSession sessConsume = sf.createSession(false, true, true, false);
-   //      
-   // sessConsume.createQueue(ADDRESS, new SimpleString(threadNum + "sub"), null, false, false);
-   //
-   // final int numMessages = 100;
-   //
-   // ClientProducer producer = sessSend.createProducer(ADDRESS);
-   //
-   // sendMessages(sessSend, producer, numMessages, threadNum);
-   //      
-   // ClientConsumer browser = sessConsume.createConsumer(new SimpleString(threadNum + "sub"),
-   // null, false, true);
-   //      
-   // Map<Integer, Integer> consumerCounts = new HashMap<Integer, Integer>();
-   //      
-   // for (int i = 0; i < numMessages; i++)
-   // {
-   // ClientMessage msg = browser.receive(RECEIVE_TIMEOUT);
-   //
-   // assertNotNull(msg);
-   //
-   // int tn = (Integer)msg.getProperty(new SimpleString("threadnum"));
-   // int cnt = (Integer)msg.getProperty(new SimpleString("count"));
-   //
-   // Integer c = consumerCounts.get(tn);
-   // if (c == null)
-   // {
-   // c = new Integer(cnt);
-   // }
-   //
-   // if (cnt != c.intValue())
-   // {
-   // throw new Exception("Invalid count, expected " + c + " got " + cnt);
-   // }
-   //         
-   // c++;
-   //         
-   // //Wrap
-   // if (c == numMessages)
-   // {
-   // c = 0;
-   // }
-   //         
-   // consumerCounts.put(tn, c);
-   //
-   // msg.acknowledge();
-   // }
-   //
-   // sessConsume.close();
-   //      
-   // sessConsume = sf.createSession(false, true, true, false);
-   //      
-   // browser = sessConsume.createConsumer(new SimpleString(threadNum + "sub"),
-   // null, false, true);
-   //      
-   // //Messages should still be there
-   //      
-   // consumerCounts.clear();
-   //      
-   // for (int i = 0; i < numMessages; i++)
-   // {
-   // ClientMessage msg = browser.receive(RECEIVE_TIMEOUT);
-   //
-   // assertNotNull(msg);
-   //
-   // int tn = (Integer)msg.getProperty(new SimpleString("threadnum"));
-   // int cnt = (Integer)msg.getProperty(new SimpleString("count"));
-   //
-   // Integer c = consumerCounts.get(tn);
-   // if (c == null)
-   // {
-   // c = new Integer(cnt);
-   // }
-   //
-   // if (cnt != c.intValue())
-   // {
-   // throw new Exception("Invalid count, expected " + c + " got " + cnt);
-   // }
-   //         
-   // c++;
-   //         
-   // //Wrap
-   // if (c == numMessages)
-   // {
-   // c = 0;
-   // }
-   //         
-   // consumerCounts.put(tn, c);
-   //
-   // msg.acknowledge();
-   // }
-   //      
-   // sessConsume.close();
-   //      
-   // sessSend.deleteQueue(new SimpleString(threadNum + "sub"));
-   //      
-   // sessSend.close();
-   //
-   // long end = System.currentTimeMillis();
-   //
-   // log.info("duration " + (end - start));
-   // }
-
-   protected void doTestN(final ClientSessionFactory sf, final int threadNum) throws Exception
-   {
-      ClientSession sessCreate = sf.createSession(false, true, true);
-
-      sessCreate.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
-
-      ClientSession sess = sf.createSession(false, true, true);
-
-      sess.stop();
-
-      sess.start();
-
-      sess.stop();
-
-      ClientConsumer consumer = sess.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
-
-      ClientProducer producer = sess.createProducer(ADDRESS);
-
-      ClientMessage message = sess.createClientMessage(JBossTextMessage.TYPE,
-                                                       false,
-                                                       0,
-                                                       System.currentTimeMillis(),
-                                                       (byte)1);
-      message.getBody().flip();
-
-      producer.send(message);
-
-      sess.start();
-
-      ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
-
-      assertNotNull(message2);
-
-      message2.acknowledge();
-
-      sess.stop();
-
-      sess.start();
-
-      sess.close();
-
-      sessCreate.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
-
-      sessCreate.close();
-   }
-
-   protected int getNumIterations()
-   {
-      return 20;
-   }
-
-   protected void setUp() throws Exception
-   {
-      super.setUp();
-
-      log.info("************ Starting test " + this.getName());
-
-      timer = new Timer();
-   }
-
-   protected void tearDown() throws Exception
-   {
-      log.info("************* Ending test " + this.getName());
-
-      if (liveService != null && liveService.isStarted())
-      {
-         liveService.stop();
-      }
-      if (backupService != null && backupService.isStarted())
-      {
-         backupService.stop();
-      }
-      timer.cancel();
-
-      super.tearDown();
-   }
-
-   // Private -------------------------------------------------------
-
-   private void runTestMultipleThreads(final RunnableT runnable, final int numThreads) throws Exception
-   {
-      final int numIts = getNumIterations();
-
-      for (int its = 0; its < numIts; its++)
-      {
-         log.info("************ ITERATION: " + its);
-         start();
-
-         final ClientSessionFactoryInternal sf = new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"),
-                                                                              new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
-                                                                                                         backupParams),
-                                                                              0,
-                                                                              1,
-                                                                              ClientSessionFactoryImpl.DEFAULT_MAX_RETRIES_BEFORE_FAILOVER,
-                                                                              ClientSessionFactoryImpl.DEFAULT_MAX_RETRIES_AFTER_FAILOVER);
-
-         sf.setSendWindowSize(32 * 1024);
-
-         ClientSession session = sf.createSession(false, false, false);
-
-         Failer failer = startFailer(1000, session);
-
-         class Runner extends Thread
-         {
-            private volatile Throwable throwable;
-
-            private final RunnableT test;
-
-            private final int threadNum;
-
-            Runner(final RunnableT test, final int threadNum)
-            {
-               this.test = test;
-
-               this.threadNum = threadNum;
-            }
-
-            public void run()
-            {
-               try
-               {
-                  test.run(sf, threadNum);
-               }
-               catch (Throwable t)
-               {
-                  throwable = t;
-
-                  log.error("Failed to run test", t);
-               }
-            }
-         }
-
-         do
-         {
-            List<Runner> threads = new ArrayList<Runner>();
-
-            for (int i = 0; i < numThreads; i++)
-            {
-               Runner runner = new Runner(runnable, i);
-
-               threads.add(runner);
-
-               runner.start();
-            }
-
-            for (Runner thread : threads)
-            {
-               thread.join();
-
-               assertNull(thread.throwable);
-            }
-
-            runnable.checkFail();
-         }
-         while (!failer.isExecuted());
-
-         InVMConnector.resetFailures();
-
-         session.close();
-
-         assertEquals(0, sf.numSessions());
-
-         assertEquals(0, sf.numConnections());
-
-         stop();
-      }
-   }
-
-   private Failer startFailer(final long time, final ClientSession session)
-   {
-      Failer failer = new Failer(session);
-
-      timer.schedule(failer, (long)(time * Math.random()), 100);
-
-      return failer;
-   }
-
-   private void start() throws Exception
-   {
       Configuration backupConf = new ConfigurationImpl();
       backupConf.setSecurityEnabled(false);
       backupParams.put(TransportConstants.SERVER_ID_PROP_NAME, 1);
@@ -1364,224 +62,22 @@
       liveService.start();
    }
 
-   private void stop() throws Exception
+   /* (non-Javadoc)
+    * @see org.jboss.messaging.tests.integration.cluster.failover.MultiThreadRandomFailoverTestBase#setBody(org.jboss.messaging.core.client.ClientMessage)
+    */
+   @Override
+   protected void setBody(ClientMessage message) throws Exception
    {
-      assertEquals(0, backupService.getServer().getRemotingService().getConnections().size());
-
-      backupService.stop();
-
-      assertEquals(0, liveService.getServer().getRemotingService().getConnections().size());
-
-      liveService.stop();
-
-      assertEquals(0, InVMRegistry.instance.size());
+      message.getBody().flip();
    }
 
-   private void sendMessages(final ClientSession sessSend,
-                             final ClientProducer producer,
-                             final int numMessages,
-                             final int threadNum) throws Exception
+   /* (non-Javadoc)
+    * @see org.jboss.messaging.tests.integration.cluster.failover.MultiThreadRandomFailoverTestBase#checkSize(org.jboss.messaging.core.client.ClientMessage)
+    */
+   @Override
+   protected boolean checkSize(ClientMessage message)
    {
-      for (int i = 0; i < numMessages; i++)
-      {
-         ClientMessage message = sessSend.createClientMessage(JBossTextMessage.TYPE,
-                                                              false,
-                                                              0,
-                                                              System.currentTimeMillis(),
-                                                              (byte)1);
-         message.putIntProperty(new SimpleString("threadnum"), threadNum);
-         message.putIntProperty(new SimpleString("count"), i);
-         message.getBody().flip();
-         producer.send(message);
-      }
+      return 0 == message.getBody().limit();
    }
 
-   private void consumeMessages(final Set<ClientConsumer> consumers, final int numMessages, final int threadNum) throws Exception
-   {
-      // We make sure the messages arrive in the order they were sent from a particular producer
-      Map<ClientConsumer, Map<Integer, Integer>> counts = new HashMap<ClientConsumer, Map<Integer, Integer>>();
-
-      for (int i = 0; i < numMessages; i++)
-      {
-         for (ClientConsumer consumer : consumers)
-         {
-            Map<Integer, Integer> consumerCounts = counts.get(consumer);
-
-            if (consumerCounts == null)
-            {
-               consumerCounts = new HashMap<Integer, Integer>();
-               counts.put(consumer, consumerCounts);
-            }
-
-            ClientMessage msg = consumer.receive(RECEIVE_TIMEOUT);
-
-            assertNotNull(msg);
-
-            int tn = (Integer)msg.getProperty(new SimpleString("threadnum"));
-            int cnt = (Integer)msg.getProperty(new SimpleString("count"));
-
-            // log.info("Got message " + tn + ":" + cnt);
-
-            Integer c = consumerCounts.get(tn);
-            if (c == null)
-            {
-               c = new Integer(cnt);
-            }
-
-            if (tn == threadNum && cnt != c.intValue())
-            {
-               throw new Exception("Invalid count, expected " + tn + ": " + c + " got " + cnt);
-            }
-
-            c++;
-
-            // Wrap
-            if (c == numMessages)
-            {
-               c = 0;
-            }
-
-            consumerCounts.put(tn, c);
-
-            msg.acknowledge();
-         }
-      }
-   }
-
-   // Inner classes -------------------------------------------------
-
-   private class Failer extends TimerTask
-   {
-      private final ClientSession session;
-
-      private boolean executed;
-
-      public Failer(final ClientSession session)
-      {
-         this.session = session;
-      }
-
-      public synchronized void run()
-      {
-         log.info("** Failing connection");
-
-         RemotingConnectionImpl conn = (RemotingConnectionImpl)((ClientSessionImpl)session).getConnection();
-
-         InVMConnector.numberOfFailures = 1;
-         InVMConnector.failOnCreateConnection = true;
-         conn.fail(new MessagingException(MessagingException.NOT_CONNECTED, "blah"));
-
-         log.info("** Fail complete");
-
-         cancel();
-
-         executed = true;
-      }
-
-      public synchronized boolean isExecuted()
-      {
-         return executed;
-      }
-   }
-
-   private abstract class RunnableT extends Thread
-   {
-      private volatile String failReason;
-
-      private volatile Throwable throwable;
-
-      public void setFailed(final String reason, final Throwable throwable)
-      {
-         this.failReason = reason;
-         this.throwable = throwable;
-      }
-
-      public void checkFail()
-      {
-         if (throwable != null)
-         {
-            log.error("Test failed: " + failReason, throwable);
-         }
-         if (failReason != null)
-         {
-            fail(failReason);
-         }
-      }
-
-      public abstract void run(final ClientSessionFactory sf, final int threadNum) throws Exception;
-   }
-
-   private class MyHandler implements MessageHandler
-   {
-      final CountDownLatch latch = new CountDownLatch(1);
-
-      private Map<Integer, Integer> counts = new HashMap<Integer, Integer>();
-
-      volatile String failure;
-
-      final int tn;
-
-      final int numMessages;
-
-      volatile boolean done;
-
-      MyHandler(final int threadNum, final int numMessages)
-      {
-         this.tn = threadNum;
-
-         this.numMessages = numMessages;
-      }
-
-      public void onMessage(ClientMessage message)
-      {
-         try
-         {
-            message.acknowledge();
-         }
-         catch (MessagingException me)
-         {
-            log.error("Failed to process", me);
-         }
-
-         if (done)
-         {
-            return;
-         }
-
-         int threadNum = (Integer)message.getProperty(new SimpleString("threadnum"));
-         int cnt = (Integer)message.getProperty(new SimpleString("count"));
-
-         Integer c = counts.get(threadNum);
-         if (c == null)
-         {
-            c = new Integer(cnt);
-         }
-
-         // log.info(System.identityHashCode(this) + " consumed message " + threadNum + ":" + cnt);
-
-         if (tn == threadNum && cnt != c.intValue())
-         {
-            failure = "Invalid count, expected " + threadNum + ":" + c + " got " + cnt;
-            log.error(failure);
-
-            latch.countDown();
-         }
-
-         if (tn == threadNum && c == numMessages - 1)
-         {
-            done = true;
-            latch.countDown();
-         }
-
-         c++;
-         // Wrap around at numMessages
-         if (c == numMessages)
-         {
-            c = 0;
-         }
-
-         counts.put(threadNum, c);
-
-      }
-   }
 }

Added: trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadRandomFailoverTestBase.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadRandomFailoverTestBase.java	                        (rev 0)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/cluster/failover/MultiThreadRandomFailoverTestBase.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -0,0 +1,1619 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.tests.integration.cluster.failover;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.jboss.messaging.core.client.ClientConsumer;
+import org.jboss.messaging.core.client.ClientMessage;
+import org.jboss.messaging.core.client.ClientProducer;
+import org.jboss.messaging.core.client.ClientSession;
+import org.jboss.messaging.core.client.ClientSessionFactory;
+import org.jboss.messaging.core.client.MessageHandler;
+import org.jboss.messaging.core.client.impl.ClientSessionFactoryImpl;
+import org.jboss.messaging.core.client.impl.ClientSessionFactoryInternal;
+import org.jboss.messaging.core.client.impl.ClientSessionImpl;
+import org.jboss.messaging.core.config.TransportConfiguration;
+import org.jboss.messaging.core.exception.MessagingException;
+import org.jboss.messaging.core.logging.Logger;
+import org.jboss.messaging.core.remoting.impl.RemotingConnectionImpl;
+import org.jboss.messaging.core.remoting.impl.invm.InVMConnector;
+import org.jboss.messaging.core.remoting.impl.invm.InVMRegistry;
+import org.jboss.messaging.core.server.MessagingService;
+import org.jboss.messaging.jms.client.JBossBytesMessage;
+import org.jboss.messaging.jms.client.JBossTextMessage;
+import org.jboss.messaging.tests.util.UnitTestCase;
+import org.jboss.messaging.util.SimpleString;
+
+/**
+ * A MultiThreadRandomFailoverTestBase
+ *
+ * @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
+ * @author <a href="mailto:clebert.suconic at jboss.org">Clebert Suconic</a>
+ * 
+ *
+ */
+public abstract class MultiThreadRandomFailoverTestBase extends UnitTestCase
+{
+
+   private static final Logger log = Logger.getLogger(MultiThreadRandomFailoverTest.class);
+
+   // Constants -----------------------------------------------------
+
+   private static final int RECEIVE_TIMEOUT = 30000;
+
+   private static final int NUM_THREADS = 10;
+
+   // Attributes ----------------------------------------------------
+   protected static final SimpleString ADDRESS = new SimpleString("FailoverTestAddress");
+
+   protected MessagingService liveService;
+
+   protected MessagingService backupService;
+
+   protected final Map<String, Object> backupParams = new HashMap<String, Object>();
+
+   protected Timer timer;
+
+   // Static --------------------------------------------------------
+
+   // Constructors --------------------------------------------------
+
+   // Public --------------------------------------------------------
+
+   public void testA() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestA(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testB() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestB(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testC() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestC(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testD() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestD(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testE() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestE(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testF() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestF(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testG() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestG(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testH() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestH(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testI() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestI(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testJ() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestJ(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testK() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestK(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   public void testL() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestL(sf);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   // public void testM() throws Exception
+   // {
+   // runTestMultipleThreads(new RunnableT()
+   // {
+   // public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+   // {
+   // doTestM(sf, threadNum);
+   // }
+   // }, NUM_THREADS);
+   // }
+
+   public void testN() throws Exception
+   {
+      runTestMultipleThreads(new RunnableT()
+      {
+         @Override
+         public void run(final ClientSessionFactory sf, final int threadNum) throws Exception
+         {
+            doTestN(sf, threadNum);
+         }
+      }, NUM_THREADS, false);
+   }
+
+   // Package protected ---------------------------------------------
+
+   // Protected -----------------------------------------------------
+
+   protected abstract void start() throws Exception;
+
+   protected abstract void setBody(ClientMessage message) throws Exception;
+
+   protected abstract boolean checkSize(ClientMessage message);
+
+   protected ClientSession createAutoCommitSession(ClientSessionFactory sf) throws Exception
+   {
+      return sf.createSession(false, true, true);      
+   }
+   
+   protected ClientSession createTransactionalSession(ClientSessionFactory sf) throws Exception
+   {
+      return sf.createSession(false, false, false);      
+   }
+   
+   protected void doTestA(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = createAutoCommitSession(sf);
+
+         sessConsume.start();
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, true, true);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      Set<MyHandler> handlers = new HashSet<MyHandler>();
+
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler(threadNum, numMessages);
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(20000, TimeUnit.MILLISECONDS);
+
+         if (!ok)
+         {
+            throw new Exception("Timed out waiting for messages on handler " + System.identityHashCode(handler) +
+                                " threadnum " +
+                                threadNum);
+         }
+
+         if (handler.failure != null)
+         {
+            throw new Exception("Handler failed: " + handler.failure);
+         }
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestB(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = createAutoCommitSession(sf);
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, true, true);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      for (ClientSession session : sessions)
+      {
+         session.start();
+      }
+
+      Set<MyHandler> handlers = new HashSet<MyHandler>();
+
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler(threadNum, numMessages);
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
+
+         if (!ok)
+         {
+            throw new Exception("Timed out waiting for messages on handler " + System.identityHashCode(handler) +
+                                " threadnum " +
+                                threadNum);
+         }
+
+         if (handler.failure != null)
+         {
+            throw new Exception("Handler failed: " + handler.failure);
+         }
+      }
+
+      sessSend.close();
+
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+
+   }
+
+   protected void doTestC(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = createTransactionalSession(sf); 
+
+         sessConsume.start();
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, false, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      sessSend.rollback();
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      sessSend.commit();
+
+      Set<MyHandler> handlers = new HashSet<MyHandler>();
+
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler(threadNum, numMessages);
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
+
+         if (!ok)
+         {
+            throw new Exception("Timed out waiting for messages on handler " + System.identityHashCode(handler) +
+                                " threadnum " +
+                                threadNum);
+         }
+
+         if (handler.failure != null)
+         {
+            throw new Exception("Handler failed: " + handler.failure);
+         }
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.rollback();
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
+
+         assertTrue(ok);
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.commit();
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestD(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + " sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, false, false);
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, false, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      sessSend.rollback();
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      sessSend.commit();
+
+      for (ClientSession session : sessions)
+      {
+         session.start();
+      }
+
+      Set<MyHandler> handlers = new HashSet<MyHandler>();
+
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler(threadNum, numMessages);
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(10000, TimeUnit.MILLISECONDS);
+
+         if (!ok)
+         {
+            throw new Exception("Timed out waiting for messages on handler " + System.identityHashCode(handler) +
+                                " threadnum " +
+                                threadNum);
+         }
+
+         if (handler.failure != null)
+         {
+            throw new Exception("Handler failed: " + handler.failure);
+         }
+      }
+
+      handlers.clear();
+
+      // Set handlers to null
+      for (ClientConsumer consumer : consumers)
+      {
+         consumer.setMessageHandler(null);
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.rollback();
+      }
+
+      // New handlers
+      for (ClientConsumer consumer : consumers)
+      {
+         MyHandler handler = new MyHandler(threadNum, numMessages);
+
+         consumer.setMessageHandler(handler);
+
+         handlers.add(handler);
+      }
+
+      for (MyHandler handler : handlers)
+      {
+         boolean ok = handler.latch.await(50000, TimeUnit.MILLISECONDS);
+
+         if (!ok)
+         {
+            throw new Exception("Timed out waiting for messages on handler " + System.identityHashCode(handler) +
+                                " threadnum " +
+                                threadNum);
+         }
+
+         if (handler.failure != null)
+         {
+            throw new Exception("Handler failed on rollback: " + handler.failure);
+         }
+      }
+
+      for (ClientSession session : sessions)
+      {
+         session.commit();
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + " sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   // Now with synchronous receive()
+
+   protected void doTestE(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, true, true);
+
+         sessConsume.start();
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, true, true);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      consumeMessages(consumers, numMessages, threadNum);
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestF(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, true, true);
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, true, true);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      for (ClientSession session : sessions)
+      {
+         session.start();
+      }
+
+      consumeMessages(consumers, numMessages, threadNum);
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestG(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, false, false);
+
+         sessConsume.start();
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, false, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      sessSend.rollback();
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      sessSend.commit();
+
+      consumeMessages(consumers, numMessages, threadNum);
+
+      for (ClientSession session : sessions)
+      {
+         session.rollback();
+      }
+
+      consumeMessages(consumers, numMessages, threadNum);
+
+      for (ClientSession session : sessions)
+      {
+         session.commit();
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestH(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      long start = System.currentTimeMillis();
+
+      ClientSession s = sf.createSession(false, false, false);
+
+      final int numMessages = 100;
+
+      final int numSessions = 10;
+
+      Set<ClientConsumer> consumers = new HashSet<ClientConsumer>();
+      Set<ClientSession> sessions = new HashSet<ClientSession>();
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         ClientSession sessConsume = sf.createSession(false, false, false);
+
+         sessConsume.createQueue(ADDRESS, subName, null, false, false);
+
+         ClientConsumer consumer = sessConsume.createConsumer(subName);
+
+         consumers.add(consumer);
+
+         sessions.add(sessConsume);
+      }
+
+      ClientSession sessSend = sf.createSession(false, false, false);
+
+      ClientProducer producer = sessSend.createProducer(ADDRESS);
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      sessSend.rollback();
+
+      sendMessages(sessSend, producer, numMessages, threadNum);
+
+      sessSend.commit();
+
+      for (ClientSession session : sessions)
+      {
+         session.start();
+      }
+
+      consumeMessages(consumers, numMessages, threadNum);
+
+      for (ClientSession session : sessions)
+      {
+         session.rollback();
+      }
+
+      consumeMessages(consumers, numMessages, threadNum);
+
+      for (ClientSession session : sessions)
+      {
+         session.commit();
+      }
+
+      sessSend.close();
+      for (ClientSession session : sessions)
+      {
+         session.close();
+      }
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         SimpleString subName = new SimpleString(threadNum + "sub" + i);
+
+         s.deleteQueue(subName);
+      }
+
+      s.close();
+
+      long end = System.currentTimeMillis();
+
+      log.info("duration " + (end - start));
+   }
+
+   protected void doTestI(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      ClientSession sessCreate = sf.createSession(false, true, true);
+
+      sessCreate.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
+
+      ClientSession sess = sf.createSession(false, true, true);
+
+      sess.start();
+
+      ClientConsumer consumer = sess.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
+
+      ClientProducer producer = sess.createProducer(ADDRESS);
+
+      ClientMessage message = sess.createClientMessage(JBossTextMessage.TYPE,
+                                                       false,
+                                                       0,
+                                                       System.currentTimeMillis(),
+                                                       (byte)1);
+      message.getBody().flip();
+
+      producer.send(message);
+
+      ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
+
+      assertNotNull(message2);
+
+      message2.acknowledge();
+
+      sess.close();
+
+      sessCreate.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
+
+      sessCreate.close();
+   }
+
+   protected void doTestJ(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      ClientSession sessCreate = sf.createSession(false, true, true);
+
+      sessCreate.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
+
+      ClientSession sess = sf.createSession(false, true, true);
+
+      sess.start();
+
+      ClientConsumer consumer = sess.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
+
+      ClientProducer producer = sess.createProducer(ADDRESS);
+
+      ClientMessage message = sess.createClientMessage(JBossTextMessage.TYPE,
+                                                       false,
+                                                       0,
+                                                       System.currentTimeMillis(),
+                                                       (byte)1);
+      message.getBody().flip();
+
+      producer.send(message);
+
+      ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
+
+      assertNotNull(message2);
+
+      message2.acknowledge();
+
+      sess.close();
+
+      sessCreate.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
+
+      sessCreate.close();
+   }
+
+   protected void doTestK(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      ClientSession s = sf.createSession(false, false, false);
+
+      s.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
+
+      final int numConsumers = 100;
+
+      for (int i = 0; i < numConsumers; i++)
+      {
+         ClientConsumer consumer = s.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
+
+         consumer.close();
+      }
+
+      s.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
+
+      s.close();
+   }
+
+   protected void doTestL(final ClientSessionFactory sf) throws Exception
+   {
+      ClientSession s = sf.createSession(false, false, false);
+
+      final int numSessions = 100;
+
+      for (int i = 0; i < numSessions; i++)
+      {
+         ClientSession session = sf.createSession(false, false, false);
+
+         session.close();
+      }
+
+      s.close();
+   }
+
+   // Browsers
+   // FIXME - this test won't work until we use a proper iterator for browsing a queue.
+   // Making a copy of the queue for a browser consumer doesn't work well with replication since
+   // When replicating the create consumer (browser) to the backup, when executed on the backup the
+   // backup may have different messages in its queue since been added on different threads.
+   // So when replicating deliveries they may not be found.
+   // https://jira.jboss.org/jira/browse/JBMESSAGING-1433
+   // protected void doTestM(final ClientSessionFactory sf, final int threadNum) throws Exception
+   // {
+   // long start = System.currentTimeMillis();
+   //
+   // ClientSession sessSend = sf.createSession(false, true, true, false);
+   //      
+   // ClientSession sessConsume = sf.createSession(false, true, true, false);
+   //      
+   // sessConsume.createQueue(ADDRESS, new SimpleString(threadNum + "sub"), null, false, false);
+   //
+   // final int numMessages = 100;
+   //
+   // ClientProducer producer = sessSend.createProducer(ADDRESS);
+   //
+   // sendMessages(sessSend, producer, numMessages, threadNum);
+   //      
+   // ClientConsumer browser = sessConsume.createConsumer(new SimpleString(threadNum + "sub"),
+   // null, false, true);
+   //      
+   // Map<Integer, Integer> consumerCounts = new HashMap<Integer, Integer>();
+   //      
+   // for (int i = 0; i < numMessages; i++)
+   // {
+   // ClientMessage msg = browser.receive(RECEIVE_TIMEOUT);
+   //
+   // assertNotNull(msg);
+   //
+   // int tn = (Integer)msg.getProperty(new SimpleString("threadnum"));
+   // int cnt = (Integer)msg.getProperty(new SimpleString("count"));
+   //
+   // Integer c = consumerCounts.get(tn);
+   // if (c == null)
+   // {
+   // c = new Integer(cnt);
+   // }
+   //
+   // if (cnt != c.intValue())
+   // {
+   // throw new Exception("Invalid count, expected " + c + " got " + cnt);
+   // }
+   //         
+   // c++;
+   //         
+   // //Wrap
+   // if (c == numMessages)
+   // {
+   // c = 0;
+   // }
+   //         
+   // consumerCounts.put(tn, c);
+   //
+   // msg.acknowledge();
+   // }
+   //
+   // sessConsume.close();
+   //      
+   // sessConsume = sf.createSession(false, true, true, false);
+   //      
+   // browser = sessConsume.createConsumer(new SimpleString(threadNum + "sub"),
+   // null, false, true);
+   //      
+   // //Messages should still be there
+   //      
+   // consumerCounts.clear();
+   //      
+   // for (int i = 0; i < numMessages; i++)
+   // {
+   // ClientMessage msg = browser.receive(RECEIVE_TIMEOUT);
+   //
+   // assertNotNull(msg);
+   //
+   // int tn = (Integer)msg.getProperty(new SimpleString("threadnum"));
+   // int cnt = (Integer)msg.getProperty(new SimpleString("count"));
+   //
+   // Integer c = consumerCounts.get(tn);
+   // if (c == null)
+   // {
+   // c = new Integer(cnt);
+   // }
+   //
+   // if (cnt != c.intValue())
+   // {
+   // throw new Exception("Invalid count, expected " + c + " got " + cnt);
+   // }
+   //         
+   // c++;
+   //         
+   // //Wrap
+   // if (c == numMessages)
+   // {
+   // c = 0;
+   // }
+   //         
+   // consumerCounts.put(tn, c);
+   //
+   // msg.acknowledge();
+   // }
+   //      
+   // sessConsume.close();
+   //      
+   // sessSend.deleteQueue(new SimpleString(threadNum + "sub"));
+   //      
+   // sessSend.close();
+   //
+   // long end = System.currentTimeMillis();
+   //
+   // log.info("duration " + (end - start));
+   // }
+
+   protected void doTestN(final ClientSessionFactory sf, final int threadNum) throws Exception
+   {
+      ClientSession sessCreate = sf.createSession(false, true, true);
+
+      sessCreate.createQueue(ADDRESS, new SimpleString(threadNum + ADDRESS.toString()), null, false, false);
+
+      ClientSession sess = sf.createSession(false, true, true);
+
+      sess.stop();
+
+      sess.start();
+
+      sess.stop();
+
+      ClientConsumer consumer = sess.createConsumer(new SimpleString(threadNum + ADDRESS.toString()));
+
+      ClientProducer producer = sess.createProducer(ADDRESS);
+
+      ClientMessage message = sess.createClientMessage(JBossTextMessage.TYPE,
+                                                       false,
+                                                       0,
+                                                       System.currentTimeMillis(),
+                                                       (byte)1);
+      message.getBody().flip();
+
+      producer.send(message);
+
+      sess.start();
+
+      ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
+
+      assertNotNull(message2);
+
+      message2.acknowledge();
+
+      sess.stop();
+
+      sess.start();
+
+      sess.close();
+
+      sessCreate.deleteQueue(new SimpleString(threadNum + ADDRESS.toString()));
+
+      sessCreate.close();
+   }
+
+   protected int getNumIterations()
+   {
+      return 20;
+   }
+
+   @Override
+   protected void setUp() throws Exception
+   {
+      super.setUp();
+
+      log.info("************ Starting test " + getName());
+
+      timer = new Timer();
+   }
+
+   @Override
+   protected void tearDown() throws Exception
+   {
+      log.info("************* Ending test " + getName());
+
+      if (liveService != null && liveService.isStarted())
+      {
+         liveService.stop();
+      }
+      if (backupService != null && backupService.isStarted())
+      {
+         backupService.stop();
+      }
+      timer.cancel();
+
+      super.tearDown();
+   }
+
+   // Private -------------------------------------------------------
+
+   private void runTestMultipleThreads(final RunnableT runnable, final int numThreads, final boolean fileBased) throws Exception
+   {
+      final int numIts = getNumIterations();
+
+      for (int its = 0; its < numIts; its++)
+      {
+         log.info("************ ITERATION: " + its);
+
+         start();
+
+         final ClientSessionFactoryInternal sf = createSessionFactory();
+
+         ClientSession session = sf.createSession(false, false, false);
+
+         Failer failer = startFailer(1000, session);
+
+         class Runner extends Thread
+         {
+            private volatile Throwable throwable;
+
+            private final RunnableT test;
+
+            private final int threadNum;
+
+            Runner(final RunnableT test, final int threadNum)
+            {
+               this.test = test;
+
+               this.threadNum = threadNum;
+            }
+
+            @Override
+            public void run()
+            {
+               try
+               {
+                  test.run(sf, threadNum);
+               }
+               catch (Throwable t)
+               {
+                  throwable = t;
+
+                  log.error("Failed to run test", t);
+               }
+            }
+         }
+
+         do
+         {
+            List<Runner> threads = new ArrayList<Runner>();
+
+            for (int i = 0; i < numThreads; i++)
+            {
+               Runner runner = new Runner(runnable, i);
+
+               threads.add(runner);
+
+               runner.start();
+            }
+
+            for (Runner thread : threads)
+            {
+               thread.join();
+
+               if (thread.throwable != null)
+               {
+                  throw new Exception("Exception on thread " + thread, thread.throwable);
+               }
+            }
+
+            runnable.checkFail();
+         }
+         while (!failer.isExecuted());
+
+         InVMConnector.resetFailures();
+
+         session.close();
+
+         assertEquals(0, sf.numSessions());
+
+         assertEquals(0, sf.numConnections());
+
+         stop();
+      }
+   }
+
+   private Failer startFailer(final long time, final ClientSession session)
+   {
+      Failer failer = new Failer(session);
+
+      timer.schedule(failer, (long)(time * Math.random()), 100);
+
+      return failer;
+   }
+
+   /**
+    * @return
+    */
+   protected ClientSessionFactoryInternal createSessionFactory()
+   {
+      final ClientSessionFactoryInternal sf = new ClientSessionFactoryImpl(new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory"),
+                                                                           new TransportConfiguration("org.jboss.messaging.core.remoting.impl.invm.InVMConnectorFactory",
+                                                                                                      backupParams),
+                                                                           0,
+                                                                           1,
+                                                                           ClientSessionFactoryImpl.DEFAULT_MAX_RETRIES_BEFORE_FAILOVER,
+                                                                           ClientSessionFactoryImpl.DEFAULT_MAX_RETRIES_AFTER_FAILOVER);
+
+      sf.setSendWindowSize(32 * 1024);
+      return sf;
+   }
+
+   private void stop() throws Exception
+   {
+      assertEquals(0, backupService.getServer().getRemotingService().getConnections().size());
+
+      backupService.stop();
+
+      assertEquals(0, liveService.getServer().getRemotingService().getConnections().size());
+
+      liveService.stop();
+
+      assertEquals(0, InVMRegistry.instance.size());
+   }
+
+   private void sendMessages(final ClientSession sessSend,
+                             final ClientProducer producer,
+                             final int numMessages,
+                             final int threadNum) throws Exception
+   {
+      for (int i = 0; i < numMessages; i++)
+      {
+         ClientMessage message = sessSend.createClientMessage(JBossBytesMessage.TYPE,
+                                                              false,
+                                                              0,
+                                                              System.currentTimeMillis(),
+                                                              (byte)1);
+         message.putIntProperty(new SimpleString("threadnum"), threadNum);
+         message.putIntProperty(new SimpleString("count"), i);
+         setBody(message);
+         producer.send(message);
+      }
+   }
+
+   private void consumeMessages(final Set<ClientConsumer> consumers, final int numMessages, final int threadNum) throws Exception
+   {
+      // We make sure the messages arrive in the order they were sent from a particular producer
+      Map<ClientConsumer, Map<Integer, Integer>> counts = new HashMap<ClientConsumer, Map<Integer, Integer>>();
+
+      for (int i = 0; i < numMessages; i++)
+      {
+         for (ClientConsumer consumer : consumers)
+         {
+            Map<Integer, Integer> consumerCounts = counts.get(consumer);
+
+            if (consumerCounts == null)
+            {
+               consumerCounts = new HashMap<Integer, Integer>();
+               counts.put(consumer, consumerCounts);
+            }
+
+            ClientMessage msg = consumer.receive(RECEIVE_TIMEOUT);
+
+            assertNotNull(msg);
+
+            int tn = (Integer)msg.getProperty(new SimpleString("threadnum"));
+            int cnt = (Integer)msg.getProperty(new SimpleString("count"));
+
+            // log.info("Got message " + tn + ":" + cnt);
+
+            Integer c = consumerCounts.get(tn);
+            if (c == null)
+            {
+               c = new Integer(cnt);
+            }
+
+            if (tn == threadNum && cnt != c.intValue())
+            {
+               throw new Exception("Invalid count, expected " + tn + ": " + c + " got " + cnt);
+            }
+
+            c++;
+
+            // Wrap
+            if (c == numMessages)
+            {
+               c = 0;
+            }
+
+            consumerCounts.put(tn, c);
+
+            msg.acknowledge();
+         }
+      }
+   }
+
+   // Inner classes -------------------------------------------------
+
+   private class Failer extends TimerTask
+   {
+      private final ClientSession session;
+
+      private boolean executed;
+
+      public Failer(final ClientSession session)
+      {
+         this.session = session;
+      }
+
+      @Override
+      public synchronized void run()
+      {
+         log.info("** Failing connection");
+
+         RemotingConnectionImpl conn = (RemotingConnectionImpl)((ClientSessionImpl)session).getConnection();
+
+         InVMConnector.numberOfFailures = 1;
+         InVMConnector.failOnCreateConnection = true;
+         conn.fail(new MessagingException(MessagingException.NOT_CONNECTED, "blah"));
+
+         log.info("** Fail complete");
+
+         cancel();
+
+         executed = true;
+      }
+
+      public synchronized boolean isExecuted()
+      {
+         return executed;
+      }
+   }
+
+   private abstract class RunnableT extends Thread
+   {
+      private volatile String failReason;
+
+      private volatile Throwable throwable;
+
+      public void setFailed(final String reason, final Throwable throwable)
+      {
+         failReason = reason;
+         this.throwable = throwable;
+      }
+
+      public void checkFail()
+      {
+         if (throwable != null)
+         {
+            log.error("Test failed: " + failReason, throwable);
+         }
+         if (failReason != null)
+         {
+            fail(failReason);
+         }
+      }
+
+      public abstract void run(final ClientSessionFactory sf, final int threadNum) throws Exception;
+   }
+
+   private class MyHandler implements MessageHandler
+   {
+      final CountDownLatch latch = new CountDownLatch(1);
+
+      private final Map<Integer, Integer> counts = new HashMap<Integer, Integer>();
+
+      volatile String failure;
+
+      final int tn;
+
+      final int numMessages;
+
+      volatile boolean done;
+
+      MyHandler(final int threadNum, final int numMessages)
+      {
+         tn = threadNum;
+
+         this.numMessages = numMessages;
+      }
+
+      public void onMessage(final ClientMessage message)
+      {
+         try
+         {
+            message.acknowledge();
+         }
+         catch (MessagingException me)
+         {
+            log.error("Failed to process", me);
+         }
+
+         if (done)
+         {
+            return;
+         }
+
+         int threadNum = (Integer)message.getProperty(new SimpleString("threadnum"));
+         int cnt = (Integer)message.getProperty(new SimpleString("count"));
+
+         Integer c = counts.get(threadNum);
+         if (c == null)
+         {
+            c = new Integer(cnt);
+         }
+
+         // log.info(System.identityHashCode(this) + " consumed message " + threadNum + ":" + cnt);
+
+         if (tn == threadNum && cnt != c.intValue())
+         {
+            failure = "Invalid count, expected " + threadNum + ":" + c + " got " + cnt;
+            log.error(failure);
+
+            latch.countDown();
+         }
+
+         if (!checkSize(message))
+         {
+            failure = "Invalid size on message";
+            log.error(failure);
+            latch.countDown();
+         }
+
+         if (tn == threadNum && c == numMessages - 1)
+         {
+            done = true;
+            latch.countDown();
+         }
+
+         c++;
+         // Wrap around at numMessages
+         if (c == numMessages)
+         {
+            c = 0;
+         }
+
+         counts.put(threadNum, c);
+
+      }
+   }
+}
\ No newline at end of file

Modified: trunk/tests/src/org/jboss/messaging/tests/integration/paging/PageCrashTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/paging/PageCrashTest.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/paging/PageCrashTest.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -23,17 +23,44 @@
 package org.jboss.messaging.tests.integration.paging;
 
 import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
 import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.Executor;
 
 import org.jboss.messaging.core.client.ClientConsumer;
+import org.jboss.messaging.core.client.ClientMessage;
+import org.jboss.messaging.core.client.ClientProducer;
 import org.jboss.messaging.core.client.ClientSession;
 import org.jboss.messaging.core.client.ClientSessionFactory;
 import org.jboss.messaging.core.config.Configuration;
+import org.jboss.messaging.core.management.ManagementService;
+import org.jboss.messaging.core.management.impl.ManagementServiceImpl;
+import org.jboss.messaging.core.paging.Page;
+import org.jboss.messaging.core.paging.PagedMessage;
+import org.jboss.messaging.core.paging.PagingManager;
+import org.jboss.messaging.core.paging.PagingStore;
+import org.jboss.messaging.core.paging.impl.PagingManagerImpl;
+import org.jboss.messaging.core.paging.impl.PagingStoreFactoryNIO;
+import org.jboss.messaging.core.paging.impl.PagingStoreImpl;
+import org.jboss.messaging.core.persistence.StorageManager;
+import org.jboss.messaging.core.persistence.impl.journal.JournalStorageManager;
+import org.jboss.messaging.core.remoting.RemotingService;
+import org.jboss.messaging.core.remoting.impl.ByteBufferWrapper;
+import org.jboss.messaging.core.remoting.impl.RemotingServiceImpl;
+import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
+import org.jboss.messaging.core.security.JBMSecurityManager;
+import org.jboss.messaging.core.security.impl.JBMSecurityManagerImpl;
+import org.jboss.messaging.core.server.MessagingServer;
 import org.jboss.messaging.core.server.MessagingService;
+import org.jboss.messaging.core.server.impl.MessagingServerImpl;
+import org.jboss.messaging.core.server.impl.MessagingServiceImpl;
 import org.jboss.messaging.core.settings.impl.QueueSettings;
-import org.jboss.messaging.tests.integration.paging.remote.RemotePageCrashExecution;
 import org.jboss.messaging.tests.util.ServiceTestBase;
-import org.jboss.messaging.tests.util.SpawnedVMSupport;
+import org.jboss.messaging.util.OrderedExecutorFactory;
+import org.jboss.messaging.util.SimpleString;
 
 /**
  * This test will make sure that a failing depage won't cause duplicated messages
@@ -49,6 +76,8 @@
 
    // Constants -----------------------------------------------------
 
+   public static final SimpleString ADDRESS = new SimpleString("SimpleAddress");
+
    // Attributes ----------------------------------------------------
 
    // Static --------------------------------------------------------
@@ -59,12 +88,9 @@
 
    public void testCrashDuringDeleteFile() throws Exception
    {
-      clearData();
+      
+      pageAndFail();
 
-      Process process = SpawnedVMSupport.spawnVM(RemotePageCrashExecution.class.getCanonicalName());
-      process.waitFor();
-      assertEquals("The remote process failed, test is invalid", RemotePageCrashExecution.OK, process.exitValue());
-
       File pageDir = new File(getPageDir());
 
       File directories[] = pageDir.listFiles();
@@ -89,15 +115,11 @@
       {
          ClientSessionFactory sf = createInVMFactory();
 
-         sf.setBlockOnNonPersistentSend(true);
-         sf.setBlockOnPersistentSend(true);
-         sf.setBlockOnAcknowledge(true);
-
          ClientSession session = sf.createSession(null, null, false, true, true, false, 0);
 
          session.start();
 
-         ClientConsumer consumer = session.createConsumer(RemotePageCrashExecution.ADDRESS);
+         ClientConsumer consumer = session.createConsumer(ADDRESS);
 
          assertNull(consumer.receive(200));
 
@@ -121,7 +143,332 @@
    }
 
    // Private -------------------------------------------------------
+   
+   
+   /** This method will leave garbage on paging. 
+    *  It will not delete page files as if the server crashed right after commit, 
+    *  and before removing the file*/
+   private void pageAndFail() throws Exception
+   {
+      clearData();
+      Configuration config = createDefaultConfig();
 
+      config.setPagingMaxGlobalSizeBytes(100 * 1024);
+      config.setPagingDefaultSize(10 * 1024);
+
+      MessagingService service = newMessagingService(config);
+
+      service.start();
+
+      try
+      {
+         ClientSessionFactory sf = createInVMFactory();
+
+         // Making it synchronous, just because we want to stop sending messages as soon as the page-store becomes in page mode
+         // and we could only guarantee that by setting it to synchronous
+         sf.setBlockOnNonPersistentSend(true);
+         sf.setBlockOnPersistentSend(true);
+         sf.setBlockOnAcknowledge(true);
+
+         ClientSession session = sf.createSession(null, null, false, true, true, false, 0);
+
+         session.createQueue(ADDRESS, ADDRESS, null, true, false);
+
+         ClientProducer producer = session.createProducer(ADDRESS);
+
+         ByteBuffer ioBuffer = ByteBuffer.allocate(1024);
+
+         ClientMessage message = null;
+
+         MessagingBuffer bodyLocal = new ByteBufferWrapper(ioBuffer);
+
+         message = session.createClientMessage(true);
+         message.setBody(bodyLocal);
+
+         PagingStore store = service.getServer().getPostOffice().getPagingManager().getPageStore(ADDRESS);
+
+         int messages = 0;
+         while (!store.isPaging())
+         {
+            producer.send(message);
+            messages++;
+         }
+
+         for (int i = 0; i < 2; i++)
+         {
+            messages++;
+            producer.send(message);
+         }
+
+         session.close();
+
+         assertTrue(service.getServer().getPostOffice().getPagingManager().getGlobalSize() > 0);
+
+         session = sf.createSession(null, null, false, true, true, false, 0);
+
+         ClientConsumer consumer = session.createConsumer(ADDRESS);
+
+         session.start();
+
+         for (int i = 0; i < messages; i++)
+         {
+            ClientMessage message2 = consumer.receive(10000);
+
+            assertNotNull(message2);
+
+            message2.acknowledge();
+         }
+
+         consumer.close();
+
+         session.close();
+
+         assertEquals(0, service.getServer().getPostOffice().getPagingManager().getGlobalSize());
+
+      }
+      finally
+      {
+         try
+         {
+            service.stop();
+         }
+         catch (Throwable ignored)
+         {
+         }
+      }
+   }
+
+   // Package protected ---------------------------------------------
+
+   // Protected -----------------------------------------------------
+
+   // Private -------------------------------------------------------
+
+   private MessagingServiceImpl newMessagingService(final Configuration configuration)
+   {
+
+      StorageManager storageManager = new JournalStorageManager(configuration);
+
+      RemotingService remotingService = new RemotingServiceImpl(configuration);
+
+      JBMSecurityManager securityManager = new JBMSecurityManagerImpl(true);
+
+      ManagementService managementService = new ManagementServiceImpl(ManagementFactory.getPlatformMBeanServer(), false);
+
+      remotingService.setManagementService(managementService);
+
+      MessagingServer server = new FailingMessagingServiceImpl();
+
+      server.setConfiguration(configuration);
+
+      server.setStorageManager(storageManager);
+
+      server.setRemotingService(remotingService);
+
+      server.setSecurityManager(securityManager);
+
+      server.setManagementService(managementService);
+
+      return new MessagingServiceImpl(server, storageManager, remotingService);
+   }
+
    // Inner classes -------------------------------------------------
 
+   /** This is hacking MessagingServerImpl, 
+    *  to make sure the server will fail right 
+    *  after before the page-file was removed */
+   class FailingMessagingServiceImpl extends MessagingServerImpl
+   {
+      /**
+       * Method could be replaced for test purposes 
+       */
+      @Override
+      protected PagingManager createPagingManager()
+      {
+         return new PagingManagerImpl(new FailurePagingStoreFactoryNIO(super.getConfiguration().getPagingDirectory(),
+                                                                       super.getConfiguration().getPagingMaxThreads()),
+                                      super.getStorageManager(),
+                                      super.getQueueSettingsRepository(),
+                                      super.getConfiguration().getPagingMaxGlobalSizeBytes(),
+                                      super.getConfiguration().getPagingDefaultSize(),
+                                      super.getConfiguration().isJournalSyncNonTransactional(),
+                                      false);
+      }
+
+      class FailurePagingStoreFactoryNIO extends PagingStoreFactoryNIO
+
+      {
+         /**
+          * @param directory
+          * @param maxThreads
+          */
+         public FailurePagingStoreFactoryNIO(final String directory, final int maxThreads)
+         {
+            super(directory, maxThreads);
+         }
+
+         // Constants -----------------------------------------------------
+
+         // Attributes ----------------------------------------------------
+
+         // Static --------------------------------------------------------
+
+         // Constructors --------------------------------------------------
+
+         // Public --------------------------------------------------------
+
+         @Override
+         public synchronized PagingStore newStore(final SimpleString destinationName, final QueueSettings settings) throws Exception
+         {
+            Field factoryField = PagingStoreFactoryNIO.class.getDeclaredField("executorFactory");
+            factoryField.setAccessible(true);
+
+            OrderedExecutorFactory factory = (OrderedExecutorFactory)factoryField.get(this);
+            return new FailingPagingStore(destinationName, settings, factory.getExecutor());
+         }
+
+         // Package protected ---------------------------------------------
+
+         // Protected -----------------------------------------------------
+
+         // Private -------------------------------------------------------
+
+         // Inner classes -------------------------------------------------
+         class FailingPagingStore extends PagingStoreImpl
+         {
+
+            /**
+             * @param pagingManager
+             * @param storageManager
+             * @param postOffice
+             * @param fileFactory
+             * @param storeFactory
+             * @param storeName
+             * @param queueSettings
+             * @param executor
+             */
+            public FailingPagingStore(final SimpleString storeName,
+                                      final QueueSettings queueSettings,
+                                      final Executor executor)
+            {
+               super(getPostOffice().getPagingManager(),
+                     getStorageManager(),
+                     getPostOffice(),
+                     null,
+                     FailurePagingStoreFactoryNIO.this,
+                     storeName,
+                     queueSettings,
+                     executor);
+            }
+
+            @Override
+            protected Page createPage(final int page) throws Exception
+            {
+
+               Page originalPage = super.createPage(page);
+
+               return new FailingPage(originalPage);
+            }
+
+         }
+
+      }
+
+      class FailingPage implements Page
+      {
+         Page delegatedPage;
+
+         /**
+          * @throws Exception
+          * @see org.jboss.messaging.core.paging.Page#close()
+          */
+         public void close() throws Exception
+         {
+            delegatedPage.close();
+         }
+
+         /**
+          * @throws Exception
+          * @see org.jboss.messaging.core.paging.Page#delete()
+          */
+         public void delete() throws Exception
+         {
+            // This will let the file stay, simulating a system failure
+         }
+
+         /**
+          * @return
+          * @see org.jboss.messaging.core.paging.Page#getNumberOfMessages()
+          */
+         public int getNumberOfMessages()
+         {
+            return delegatedPage.getNumberOfMessages();
+         }
+
+         /**
+          * @return
+          * @see org.jboss.messaging.core.paging.Page#getPageId()
+          */
+         public int getPageId()
+         {
+            return delegatedPage.getPageId();
+         }
+
+         /**
+          * @return
+          * @see org.jboss.messaging.core.paging.Page#getSize()
+          */
+         public int getSize()
+         {
+            return delegatedPage.getSize();
+         }
+
+         /**
+          * @throws Exception
+          * @see org.jboss.messaging.core.paging.Page#open()
+          */
+         public void open() throws Exception
+         {
+            delegatedPage.open();
+         }
+
+         /**
+          * @return
+          * @throws Exception
+          * @see org.jboss.messaging.core.paging.Page#read()
+          */
+         public List<PagedMessage> read() throws Exception
+         {
+            return delegatedPage.read();
+         }
+
+         /**
+          * @throws Exception
+          * @see org.jboss.messaging.core.paging.Page#sync()
+          */
+         public void sync() throws Exception
+         {
+            delegatedPage.sync();
+         }
+
+         /**
+          * @param message
+          * @throws Exception
+          * @see org.jboss.messaging.core.paging.Page#write(org.jboss.messaging.core.paging.PagedMessage)
+          */
+         public void write(final PagedMessage message) throws Exception
+         {
+            delegatedPage.write(message);
+         }
+
+         public FailingPage(final Page delegatePage)
+         {
+            delegatedPage = delegatePage;
+         }
+      }
+
+   }
+
+   // Inner classes -------------------------------------------------
+
 }

Modified: trunk/tests/src/org/jboss/messaging/tests/integration/paging/PagingManagerIntegrationTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/paging/PagingManagerIntegrationTest.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/paging/PagingManagerIntegrationTest.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -72,7 +72,8 @@
                                                             queueSettings,
                                                             -1,
                                                             1024 * 1024,
-                                                            true);
+                                                            true,
+                                                            false);
 
       managerImpl.start();
 
@@ -122,6 +123,7 @@
                                                             queueSettings,
                                                             -1,
                                                             1024 * 1024,
+                                                            false,
                                                             false);
       managerImpl.start();
 

Modified: trunk/tests/src/org/jboss/messaging/tests/integration/paging/PagingServiceIntegrationTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/paging/PagingServiceIntegrationTest.java	2009-01-21 21:41:57 UTC (rev 5677)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/paging/PagingServiceIntegrationTest.java	2009-01-21 21:43:23 UTC (rev 5678)
@@ -637,12 +637,20 @@
       }
 
    }
-
-
+   
    public void testPageMultipleDestinations() throws Exception
    {
-      clearData();
+      internalTestPageMultipleDestinations(false);
+   }
 
+   
+   public void testPageMultipleDestinationsTransacted() throws Exception
+   {
+      internalTestPageMultipleDestinations(true);
+   }
+   
+   private void internalTestPageMultipleDestinations(boolean transacted) throws Exception
+   {
       Configuration config = createDefaultConfig();
 
       final int MAX_SIZE = 90 * 1024; // this must be lower than minlargeMessageSize on the SessionFactory
@@ -666,7 +674,7 @@
          sf.setBlockOnPersistentSend(true);
          sf.setBlockOnAcknowledge(true);
 
-         ClientSession session = sf.createSession(null, null, false, true, true, false, 0);
+         ClientSession session = sf.createSession(null, null, false, !transacted, true, false, 0);
 
          for (int i = 0; i < NUMBER_OF_BINDINGS; i++)
          {
@@ -687,6 +695,11 @@
          for (int i = 0; i < NUMBER_OF_MESSAGES; i++)
          {
             producer.send(message);
+            
+            if (transacted) 
+            {
+               session.commit();
+            }
          }
 
          session.close();




More information about the jboss-cvs-commits mailing list