[infinispan-commits] Infinispan SVN: r673 - in trunk: core/src/main/java/org/infinispan/commands and 20 other directories.

infinispan-commits at lists.jboss.org infinispan-commits at lists.jboss.org
Thu Aug 13 11:16:26 EDT 2009


Author: manik.surtani at jboss.com
Date: 2009-08-13 11:16:24 -0400 (Thu, 13 Aug 2009)
New Revision: 673

Added:
   trunk/core/src/main/java/org/infinispan/commands/control/GetConsistentHashCommand.java
   trunk/core/src/main/java/org/infinispan/commands/control/InstallConsistentHashCommand.java
   trunk/core/src/main/java/org/infinispan/commands/control/JoinCompleteCommand.java
   trunk/core/src/main/java/org/infinispan/commands/control/PullStateCommand.java
   trunk/core/src/main/java/org/infinispan/commands/control/PushStateCommand.java
   trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcCommand.java
   trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcInvokingCommand.java
   trunk/core/src/main/java/org/infinispan/distribution/AbstractConsistentHash.java
   trunk/core/src/main/java/org/infinispan/distribution/JoinTask.java
   trunk/core/src/main/java/org/infinispan/distribution/LeaveTask.java
   trunk/core/src/main/java/org/infinispan/distribution/RecvLeaveTask.java
   trunk/core/src/main/java/org/infinispan/distribution/RehashHandler.java
   trunk/core/src/main/java/org/infinispan/distribution/RehashTask.java
   trunk/core/src/main/java/org/infinispan/distribution/SendLeaveTask.java
   trunk/core/src/main/java/org/infinispan/distribution/TransactionLogger.java
   trunk/core/src/main/java/org/infinispan/distribution/TransactionLoggerImpl.java
   trunk/core/src/main/java/org/infinispan/distribution/UnionConsistentHash.java
   trunk/core/src/main/java/org/infinispan/remoting/responses/UnsureResponse.java
   trunk/core/src/test/java/org/infinispan/distribution/RehashJoinTest.java
Removed:
   trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcCommand.java
Modified:
   trunk/core/src/main/java/org/infinispan/AbstractDelegatingAdvancedCache.java
   trunk/core/src/main/java/org/infinispan/AbstractDelegatingCache.java
   trunk/core/src/main/java/org/infinispan/commands/CommandsFactory.java
   trunk/core/src/main/java/org/infinispan/commands/CommandsFactoryImpl.java
   trunk/core/src/main/java/org/infinispan/commands/RemoteCommandFactory.java
   trunk/core/src/main/java/org/infinispan/commands/remote/ClusteredGetCommand.java
   trunk/core/src/main/java/org/infinispan/commands/remote/MultipleRpcCommand.java
   trunk/core/src/main/java/org/infinispan/commands/remote/SingleRpcCommand.java
   trunk/core/src/main/java/org/infinispan/distribution/ConsistentHash.java
   trunk/core/src/main/java/org/infinispan/distribution/DefaultConsistentHash.java
   trunk/core/src/main/java/org/infinispan/distribution/DistributionManager.java
   trunk/core/src/main/java/org/infinispan/distribution/DistributionManagerImpl.java
   trunk/core/src/main/java/org/infinispan/loaders/decorators/SingletonStore.java
   trunk/core/src/main/java/org/infinispan/marshall/Ids.java
   trunk/core/src/main/java/org/infinispan/marshall/jboss/ConstantObjectTable.java
   trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifier.java
   trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierImpl.java
   trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/event/EventImpl.java
   trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/event/ViewChangedEvent.java
   trunk/core/src/main/java/org/infinispan/remoting/InboundInvocationHandlerImpl.java
   trunk/core/src/main/java/org/infinispan/remoting/responses/DistributionResponseGenerator.java
   trunk/core/src/main/java/org/infinispan/remoting/rpc/RpcManager.java
   trunk/core/src/main/java/org/infinispan/remoting/rpc/RpcManagerImpl.java
   trunk/core/src/main/java/org/infinispan/remoting/transport/Transport.java
   trunk/core/src/main/java/org/infinispan/remoting/transport/jgroups/JGroupsTransport.java
   trunk/core/src/main/java/org/infinispan/util/Util.java
   trunk/core/src/main/java/org/infinispan/util/concurrent/ReclosableLatch.java
   trunk/core/src/test/java/org/infinispan/distribution/AsyncAPIAsyncDistTest.java
   trunk/core/src/test/java/org/infinispan/distribution/AsyncAPISyncDistTest.java
   trunk/core/src/test/java/org/infinispan/distribution/BaseDistCacheStoreTest.java
   trunk/core/src/test/java/org/infinispan/distribution/BaseDistFunctionalTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DefaultConsistentHashTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistAsyncFuncTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistAsyncTxFuncTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistAsyncTxUnsafeFuncTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistAsyncUnsafeFuncTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistSkipRemoteLookupTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistSyncCacheStoreNotSharedTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistSyncCacheStoreSharedTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistSyncFuncTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistSyncTxFuncTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistSyncTxUnsafeFuncTest.java
   trunk/core/src/test/java/org/infinispan/distribution/DistSyncUnsafeFuncTest.java
   trunk/core/src/test/java/org/infinispan/distribution/MagicKeyTest.java
   trunk/core/src/test/java/org/infinispan/marshall/MarshallersTest.java
   trunk/core/src/test/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierImplTest.java
   trunk/core/src/test/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierTest.java
   trunk/core/src/test/java/org/infinispan/tx/ReplDeadlockDetectionTest.java
   trunk/gui-demo/src/main/java/org/infinispan/demo/InfinispanDemo.java
Log:
[ISPN-65] (Rehashing ('State Transfer') of entries when caches join/leave group) initial version of rehashing for joiners + tests

Modified: trunk/core/src/main/java/org/infinispan/AbstractDelegatingAdvancedCache.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/AbstractDelegatingAdvancedCache.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/AbstractDelegatingAdvancedCache.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -21,7 +21,7 @@
  * @author Mircea.Markus at jboss.com
  * @see org.infinispan.AbstractDelegatingCache
  */
-public class AbstractDelegatingAdvancedCache<K, V> extends AbstractDelegatingCache<K, V> implements AdvancedCache<K, V> {
+public abstract class AbstractDelegatingAdvancedCache<K, V> extends AbstractDelegatingCache<K, V> implements AdvancedCache<K, V> {
 
    private AdvancedCache<K, V> cache;
 

Modified: trunk/core/src/main/java/org/infinispan/AbstractDelegatingCache.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/AbstractDelegatingCache.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/AbstractDelegatingCache.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -19,7 +19,7 @@
  * @author Mircea.Markus at jboss.com
  * @see org.infinispan.AbstractDelegatingAdvancedCache
  */
-public class AbstractDelegatingCache<K, V> implements Cache<K, V> {
+public abstract class AbstractDelegatingCache<K, V> implements Cache<K, V> {
 
    private Cache<K, V> cache;
 

Modified: trunk/core/src/main/java/org/infinispan/commands/CommandsFactory.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/CommandsFactory.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/commands/CommandsFactory.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -21,7 +21,12 @@
  */
 package org.infinispan.commands;
 
+import org.infinispan.commands.control.GetConsistentHashCommand;
+import org.infinispan.commands.control.InstallConsistentHashCommand;
+import org.infinispan.commands.control.JoinCompleteCommand;
 import org.infinispan.commands.control.LockControlCommand;
+import org.infinispan.commands.control.PullStateCommand;
+import org.infinispan.commands.control.PushStateCommand;
 import org.infinispan.commands.control.StateTransferControlCommand;
 import org.infinispan.commands.read.EntrySetCommand;
 import org.infinispan.commands.read.GetKeyValueCommand;
@@ -42,8 +47,11 @@
 import org.infinispan.commands.write.RemoveCommand;
 import org.infinispan.commands.write.ReplaceCommand;
 import org.infinispan.commands.write.WriteCommand;
+import org.infinispan.container.entries.InternalCacheValue;
+import org.infinispan.distribution.ConsistentHash;
 import org.infinispan.factories.scopes.Scope;
 import org.infinispan.factories.scopes.Scopes;
+import org.infinispan.remoting.transport.Address;
 import org.infinispan.transaction.xa.GlobalTransaction;
 
 import java.util.Collection;
@@ -111,4 +119,14 @@
    ClusteredGetCommand buildClusteredGetCommand(Object key);
 
    LockControlCommand buildLockControlCommand(Collection keys, boolean implicit);
+
+   GetConsistentHashCommand buildGetConsistentHashCommand(Address joiner);
+
+   InstallConsistentHashCommand buildInstallConsistentHashCommand(Address joiner, boolean starting);
+
+   PushStateCommand buildPushStateCommand(Address sender, Map<Object, InternalCacheValue> state);
+
+   PullStateCommand buildPullStateCommand(Address requestor, ConsistentHash newCH);
+
+   JoinCompleteCommand buildJoinCompleteCommand(Address joiner);
 }

Modified: trunk/core/src/main/java/org/infinispan/commands/CommandsFactoryImpl.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/CommandsFactoryImpl.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/commands/CommandsFactoryImpl.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -22,7 +22,12 @@
 package org.infinispan.commands;
 
 import org.infinispan.Cache;
+import org.infinispan.commands.control.GetConsistentHashCommand;
+import org.infinispan.commands.control.InstallConsistentHashCommand;
+import org.infinispan.commands.control.JoinCompleteCommand;
 import org.infinispan.commands.control.LockControlCommand;
+import org.infinispan.commands.control.PullStateCommand;
+import org.infinispan.commands.control.PushStateCommand;
 import org.infinispan.commands.control.StateTransferControlCommand;
 import org.infinispan.commands.read.EntrySetCommand;
 import org.infinispan.commands.read.GetKeyValueCommand;
@@ -44,13 +49,18 @@
 import org.infinispan.commands.write.RemoveCommand;
 import org.infinispan.commands.write.ReplaceCommand;
 import org.infinispan.commands.write.WriteCommand;
+import org.infinispan.config.Configuration;
 import org.infinispan.container.DataContainer;
+import org.infinispan.container.entries.InternalCacheValue;
 import org.infinispan.context.InvocationContextContainer;
+import org.infinispan.distribution.ConsistentHash;
 import org.infinispan.distribution.DistributionManager;
 import org.infinispan.factories.annotations.Inject;
 import org.infinispan.factories.annotations.Start;
 import org.infinispan.interceptors.InterceptorChain;
+import org.infinispan.loaders.CacheLoaderManager;
 import org.infinispan.notifications.cachelistener.CacheNotifier;
+import org.infinispan.remoting.transport.Address;
 import org.infinispan.transaction.xa.GlobalTransaction;
 import org.infinispan.transaction.xa.TransactionTable;
 
@@ -78,11 +88,14 @@
    private DistributionManager distributionManager;
    private InvocationContextContainer icc;
    private TransactionTable txTable;
+   private Configuration configuration;
+   private CacheLoaderManager cacheLoaderManager;
 
    @Inject
    public void setupDependencies(DataContainer container, CacheNotifier notifier, Cache cache,
                                  InterceptorChain interceptorChain, DistributionManager distributionManager,
-                                 InvocationContextContainer icc, TransactionTable txTable) {
+                                 InvocationContextContainer icc, TransactionTable txTable, CacheLoaderManager cacheLoaderManager,
+                                 Configuration configuration) {
       this.dataContainer = container;
       this.notifier = notifier;
       this.cache = cache;
@@ -90,6 +103,8 @@
       this.distributionManager = distributionManager;
       this.icc = icc;
       this.txTable = txTable;
+      this.configuration = configuration;
+      this.cacheLoaderManager = cacheLoaderManager;
    }
 
    @Start(priority = 1)
@@ -259,10 +274,46 @@
             LockControlCommand lcc = (LockControlCommand) c;
             lcc.init(interceptorChain, icc, txTable);
             break;
+         case GetConsistentHashCommand.COMMAND_ID:
+            GetConsistentHashCommand gchc = (GetConsistentHashCommand) c;
+            gchc.initialize(distributionManager);
+            break;
+         case InstallConsistentHashCommand.COMMAND_ID:
+            InstallConsistentHashCommand ichc = (InstallConsistentHashCommand) c;
+            ichc.initialize(distributionManager);
+            break;
+         case PullStateCommand.COMMAND_ID:
+            PullStateCommand psc = (PullStateCommand) c;
+            psc.init(dataContainer, cacheLoaderManager, distributionManager, configuration);
+            break;
+         case JoinCompleteCommand.COMMAND_ID:
+            JoinCompleteCommand jcc = (JoinCompleteCommand) c;
+            jcc.init(distributionManager);
+            break;
       }
    }
 
    public LockControlCommand buildLockControlCommand(Collection keys, boolean implicit) {
       return new LockControlCommand(keys, cacheName, implicit);
    }
+
+   public GetConsistentHashCommand buildGetConsistentHashCommand(Address joiner) {
+      return new GetConsistentHashCommand(cacheName, joiner);
+   }
+
+   public InstallConsistentHashCommand buildInstallConsistentHashCommand(Address joiner, boolean starting) {
+      return new InstallConsistentHashCommand(cacheName, joiner, starting);
+   }
+
+   public PushStateCommand buildPushStateCommand(Address sender, Map<Object, InternalCacheValue> state) {
+      return new PushStateCommand(cacheName, sender, state);
+   }
+
+   public PullStateCommand buildPullStateCommand(Address requestor, ConsistentHash newCH) {
+      return new PullStateCommand(cacheName, requestor, newCH);
+   }
+
+   public JoinCompleteCommand buildJoinCompleteCommand(Address joiner) {
+      return new JoinCompleteCommand(cacheName, joiner);
+   }
 }

Modified: trunk/core/src/main/java/org/infinispan/commands/RemoteCommandFactory.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/RemoteCommandFactory.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/commands/RemoteCommandFactory.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,7 +1,12 @@
 package org.infinispan.commands;
 
 import org.infinispan.CacheException;
+import org.infinispan.commands.control.GetConsistentHashCommand;
+import org.infinispan.commands.control.InstallConsistentHashCommand;
+import org.infinispan.commands.control.JoinCompleteCommand;
 import org.infinispan.commands.control.LockControlCommand;
+import org.infinispan.commands.control.PullStateCommand;
+import org.infinispan.commands.control.PushStateCommand;
 import org.infinispan.commands.control.StateTransferControlCommand;
 import org.infinispan.commands.read.GetKeyValueCommand;
 import org.infinispan.commands.remote.ClusteredGetCommand;
@@ -100,6 +105,21 @@
          case ClusteredGetCommand.COMMAND_ID:
             command = new ClusteredGetCommand();
             break;
+         case GetConsistentHashCommand.COMMAND_ID:
+            command = new GetConsistentHashCommand();
+            break;
+         case InstallConsistentHashCommand.COMMAND_ID:
+            command = new InstallConsistentHashCommand();
+            break;
+         case PushStateCommand.COMMAND_ID:
+            command = new PushStateCommand();
+            break;
+         case PullStateCommand.COMMAND_ID:
+            command = new PullStateCommand(transport);
+            break;
+         case JoinCompleteCommand.COMMAND_ID:
+            command = new JoinCompleteCommand();
+            break;
          default:
             throw new CacheException("Unknown command id " + id + "!");
       }

Added: trunk/core/src/main/java/org/infinispan/commands/control/GetConsistentHashCommand.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/control/GetConsistentHashCommand.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/commands/control/GetConsistentHashCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,59 @@
+package org.infinispan.commands.control;
+
+import org.infinispan.commands.remote.BaseRpcCommand;
+import org.infinispan.context.InvocationContext;
+import org.infinispan.distribution.DistributionManager;
+import org.infinispan.marshall.Ids;
+import org.infinispan.marshall.Marshallable;
+import org.infinispan.marshall.exts.ReplicableCommandExternalizer;
+import org.infinispan.remoting.transport.Address;
+
+/**
+ * Retrieves a consistent hash instance from the distribution manager.  This command is always sent to the coordinator
+ * by a new joiner.
+ * <p/>
+ * // TODO rename to GET_ADDRESS_LIST_FROM_COORD, document accordingly
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+ at Marshallable(externalizer = ReplicableCommandExternalizer.class, id = Ids.GET_CONSISTENT_HASH_COMMAND)
+public class GetConsistentHashCommand extends BaseRpcCommand {
+
+   public static final int COMMAND_ID = 17;
+   DistributionManager distributionManager;
+   Address joiner;
+
+   public GetConsistentHashCommand() {
+   }
+
+   public GetConsistentHashCommand(String cacheName, Address joiner) {
+      super(cacheName);
+      this.joiner = joiner;
+   }
+
+   public GetConsistentHashCommand(DistributionManager distributionManager) {
+      this.distributionManager = distributionManager;
+   }
+
+   public void initialize(DistributionManager distributionManager) {
+      this.distributionManager = distributionManager;
+   }
+
+   public Object perform(InvocationContext ctx) throws Throwable {
+      return distributionManager.requestPermissionToJoin(joiner);
+   }
+
+   public byte getCommandId() {
+      return COMMAND_ID;
+   }
+
+   public Object[] getParameters() {
+      return new Object[]{cacheName, joiner};
+   }
+
+   public void setParameters(int commandId, Object[] parameters) {
+      cacheName = (String) parameters[0];
+      joiner = (Address) parameters[1];
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/commands/control/GetConsistentHashCommand.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/commands/control/InstallConsistentHashCommand.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/control/InstallConsistentHashCommand.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/commands/control/InstallConsistentHashCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,58 @@
+package org.infinispan.commands.control;
+
+import org.infinispan.commands.remote.BaseRpcCommand;
+import org.infinispan.context.InvocationContext;
+import org.infinispan.distribution.DistributionManager;
+import org.infinispan.marshall.Ids;
+import org.infinispan.marshall.Marshallable;
+import org.infinispan.marshall.exts.ReplicableCommandExternalizer;
+import org.infinispan.remoting.transport.Address;
+
+/**
+ * Installs a consistent hash in a distribution manager
+ * <p/>
+ * // TODO rename to INFORM_REHASH_ON_JOIN
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+ at Marshallable(externalizer = ReplicableCommandExternalizer.class, id = Ids.INSTALL_CONSISTENT_HASH_COMMAND)
+public class InstallConsistentHashCommand extends BaseRpcCommand {
+
+   public static final int COMMAND_ID = 18;
+   DistributionManager distributionManager;
+   Address joiner;
+   boolean starting; // if true the rehash is starting; if false it has completed.
+
+   public InstallConsistentHashCommand() {
+   }
+
+   public InstallConsistentHashCommand(String cacheName, Address joiner, boolean starting) {
+      super(cacheName);
+      this.joiner = joiner;
+      this.starting = starting;
+   }
+
+   public void initialize(DistributionManager distributionManager) {
+      this.distributionManager = distributionManager;
+   }
+
+   public Object perform(InvocationContext ctx) throws Throwable {
+      distributionManager.informRehashOnJoin(joiner, starting);
+      return null;
+   }
+
+   public byte getCommandId() {
+      return COMMAND_ID;
+   }
+
+   public Object[] getParameters() {
+      return new Object[]{cacheName, joiner, starting};
+   }
+
+   public void setParameters(int commandId, Object[] parameters) {
+      cacheName = (String) parameters[0];
+      joiner = (Address) parameters[1];
+      starting = (Boolean) parameters[2];
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/commands/control/InstallConsistentHashCommand.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/commands/control/JoinCompleteCommand.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/control/JoinCompleteCommand.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/commands/control/JoinCompleteCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,52 @@
+package org.infinispan.commands.control;
+
+import org.infinispan.commands.remote.BaseRpcCommand;
+import org.infinispan.context.InvocationContext;
+import org.infinispan.distribution.DistributionManager;
+import org.infinispan.marshall.Ids;
+import org.infinispan.marshall.Marshallable;
+import org.infinispan.marshall.exts.ReplicableCommandExternalizer;
+import org.infinispan.remoting.transport.Address;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+ at Marshallable(externalizer = ReplicableCommandExternalizer.class, id = Ids.JOIN_COMPLETE_COMMAND)
+public class JoinCompleteCommand extends BaseRpcCommand {
+   public static final int COMMAND_ID = 21;
+   Address joiner;
+   DistributionManager distributionManager;
+
+   public JoinCompleteCommand() {
+   }
+
+   public JoinCompleteCommand(String cacheName, Address joiner) {
+      super(cacheName);
+      this.joiner = joiner;
+   }
+
+   public void init(DistributionManager distributionManager) {
+      this.distributionManager = distributionManager;
+   }
+
+   public Object perform(InvocationContext ctx) throws Throwable {
+      distributionManager.notifyJoinComplete(joiner);
+      return null;
+   }
+
+   public byte getCommandId() {
+      return COMMAND_ID;
+   }
+
+   public Object[] getParameters() {
+      return new Object[]{cacheName, joiner};
+   }
+
+   public void setParameters(int commandId, Object[] parameters) {
+      cacheName = (String) parameters[0];
+      joiner = (Address) parameters[1];
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/commands/control/JoinCompleteCommand.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/commands/control/PullStateCommand.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/control/PullStateCommand.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/commands/control/PullStateCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,121 @@
+package org.infinispan.commands.control;
+
+import org.infinispan.commands.remote.BaseRpcCommand;
+import org.infinispan.config.Configuration;
+import org.infinispan.container.DataContainer;
+import org.infinispan.container.entries.InternalCacheEntry;
+import org.infinispan.container.entries.InternalCacheValue;
+import org.infinispan.context.InvocationContext;
+import org.infinispan.distribution.ConsistentHash;
+import org.infinispan.distribution.DistributionManager;
+import org.infinispan.loaders.CacheLoaderManager;
+import org.infinispan.loaders.CacheStore;
+import org.infinispan.marshall.Ids;
+import org.infinispan.marshall.Marshallable;
+import org.infinispan.marshall.exts.ReplicableCommandExternalizer;
+import org.infinispan.remoting.transport.Address;
+import org.infinispan.remoting.transport.Transport;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+ at Marshallable(externalizer = ReplicableCommandExternalizer.class, id = Ids.PULL_STATE_COMMAND)
+public class PullStateCommand extends BaseRpcCommand {
+
+   public static final int COMMAND_ID = 19;
+   Address requestor;
+   DataContainer dataContainer;
+   CacheLoaderManager clm;
+   Transport transport;
+   ConsistentHash newCH, oldCH;
+   Address self;
+   DistributionManager distributionManager;
+   Configuration configuration;
+   int numCopies;
+
+   public PullStateCommand() {
+   }
+
+   public PullStateCommand(String cacheName, Address requestor, ConsistentHash newCH) {
+      super(cacheName);
+      this.requestor = requestor;
+      this.newCH = newCH;
+   }
+
+   public PullStateCommand(Transport transport) {
+      this.transport = transport;
+   }
+
+   public void init(DataContainer dataContainer, CacheLoaderManager clm, DistributionManager distributionManager, Configuration c) {
+      this.dataContainer = dataContainer;
+      this.clm = clm;
+      this.distributionManager = distributionManager;
+      this.configuration = c;
+   }
+
+   /**
+    * @param ctx invocation context
+    * @return Should return a Map<Object, InternalCacheValue>
+    * @throws Throwable
+    */
+   public Object perform(InvocationContext ctx) throws Throwable {
+      self = transport.getAddress();
+      oldCH = distributionManager.getConsistentHash();
+      numCopies = configuration.getNumOwners();
+
+      Map<Object, InternalCacheValue> state = new HashMap<Object, InternalCacheValue>();
+      for (InternalCacheEntry ice : dataContainer) {
+         Object k = ice.getKey();
+         if (shouldAddToMap(k)) {
+            state.put(k, ice.toInternalCacheValue());
+         }
+      }
+
+      CacheStore cacheStore = getCacheStore();
+      if (cacheStore != null) {
+         for (InternalCacheEntry ice : cacheStore.loadAll()) {
+            Object k = ice.getKey();
+            if (shouldAddToMap(k) && !state.containsKey(k)) {
+               state.put(k, ice.toInternalCacheValue());
+            }
+         }
+      }
+      return state;
+   }
+
+   final boolean shouldAddToMap(Object k) {
+      // if the current address is the current "owner" of this key (in old_ch), and the requestor is in the owner list
+      // in new_ch, then add this to the map.
+      List<Address> oldOwnerList = oldCH.locate(k, numCopies);
+      if (oldOwnerList.size() > 0 && self.equals(oldOwnerList.get(0))) {
+         List<Address> newOwnerList = newCH.locate(k, numCopies);
+         if (newOwnerList.contains(requestor)) return true;
+      }
+      return false;
+   }
+
+   final CacheStore getCacheStore() {
+      return clm != null && clm.isEnabled() ? clm.getCacheStore() : null;
+   }
+
+   public byte getCommandId() {
+      return COMMAND_ID;
+   }
+
+   public Object[] getParameters() {
+      return new Object[]{cacheName, requestor, newCH};
+   }
+
+   public void setParameters(int commandId, Object[] parameters) {
+      cacheName = (String) parameters[0];
+      requestor = (Address) parameters[1];
+      newCH = (ConsistentHash) parameters[2];
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/commands/control/PullStateCommand.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/commands/control/PushStateCommand.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/control/PushStateCommand.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/commands/control/PushStateCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,53 @@
+package org.infinispan.commands.control;
+
+import org.infinispan.commands.remote.BaseRpcCommand;
+import org.infinispan.container.entries.InternalCacheValue;
+import org.infinispan.context.InvocationContext;
+import org.infinispan.marshall.Ids;
+import org.infinispan.marshall.Marshallable;
+import org.infinispan.marshall.exts.ReplicableCommandExternalizer;
+import org.infinispan.remoting.transport.Address;
+
+import java.util.Map;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+ at Marshallable(externalizer = ReplicableCommandExternalizer.class, id = Ids.PUSH_STATE_COMMAND)
+public class PushStateCommand extends BaseRpcCommand {
+
+   public static final int COMMAND_ID = 20;
+   Address sender;
+   Map<Object, InternalCacheValue> state;
+
+   public PushStateCommand() {
+   }
+
+   public PushStateCommand(String cacheName, Address sender, Map<Object, InternalCacheValue> state) {
+      super(cacheName);
+      this.sender = sender;
+      this.state = state;
+   }
+
+   public Object perform(InvocationContext ctx) throws Throwable {
+      return null;  // TODO: Customise this generated block
+   }
+
+   public byte getCommandId() {
+      return COMMAND_ID;
+   }
+
+   public Object[] getParameters() {
+      return new Object[]{cacheName, sender, state};
+   }
+
+   @SuppressWarnings("unchecked")
+   public void setParameters(int commandId, Object[] parameters) {
+      cacheName = (String) parameters[0];
+      sender = (Address) parameters[1];
+      state = (Map<Object, InternalCacheValue>) parameters[2];
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/commands/control/PushStateCommand.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Deleted: trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcCommand.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcCommand.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,51 +0,0 @@
-package org.infinispan.commands.remote;
-
-import org.infinispan.commands.ReplicableCommand;
-import org.infinispan.commands.VisitableCommand;
-import org.infinispan.interceptors.InterceptorChain;
-import org.infinispan.util.logging.Log;
-import org.infinispan.util.logging.LogFactory;
-import org.infinispan.context.InvocationContext;
-import org.infinispan.context.InvocationContextContainer;
-
-/**
- * Base class for RPC commands.
- *
- * @author Mircea.Markus at jboss.com
- */
-public abstract class BaseRpcCommand implements CacheRpcCommand {
-
-   protected InterceptorChain interceptorChain;
-   protected InvocationContextContainer icc;
-   protected String cacheName;
-
-   private static final Log log = LogFactory.getLog(BaseRpcCommand.class);
-   private static final boolean trace = log.isTraceEnabled();
-
-   protected BaseRpcCommand(String cacheName) {
-      this.cacheName = cacheName;
-   }
-
-   BaseRpcCommand() {
-   }
-
-   public String getCacheName() {
-      return cacheName;
-   }
-
-   public void init(InterceptorChain interceptorChain, InvocationContextContainer icc) {
-      this.interceptorChain = interceptorChain;
-      this.icc = icc;
-   }
-
-   protected final Object processVisitableCommand(ReplicableCommand cacheCommand) throws Throwable {
-      if (cacheCommand instanceof VisitableCommand) {
-         InvocationContext ctx = icc.createRemoteInvocationContext();
-         if (trace) log.trace("Invoking command " + cacheCommand + ", with originLocal flag set to " + ctx.isOriginLocal() + ".");
-         return interceptorChain.invoke(ctx, (VisitableCommand) cacheCommand);
-         // we only need to return values for a set of remote calls; not every call.
-      } else {
-         throw new RuntimeException("Do we still need to deal with non-visitable commands? (" + cacheCommand.getClass().getName() + ")");
-      }
-   }
-}

Added: trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcCommand.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcCommand.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,16 @@
+package org.infinispan.commands.remote;
+
+public abstract class BaseRpcCommand implements CacheRpcCommand {
+   protected String cacheName;
+
+   protected BaseRpcCommand(String cacheName) {
+      this.cacheName = cacheName;
+   }
+
+   protected BaseRpcCommand() {
+   }
+
+   public String getCacheName() {
+      return cacheName;
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcCommand.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Copied: trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcInvokingCommand.java (from rev 602, trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcCommand.java)
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcInvokingCommand.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcInvokingCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,47 @@
+package org.infinispan.commands.remote;
+
+import org.infinispan.commands.ReplicableCommand;
+import org.infinispan.commands.VisitableCommand;
+import org.infinispan.context.InvocationContext;
+import org.infinispan.context.InvocationContextContainer;
+import org.infinispan.interceptors.InterceptorChain;
+import org.infinispan.util.logging.Log;
+import org.infinispan.util.logging.LogFactory;
+
+/**
+ * Base class for RPC commands.
+ *
+ * @author Mircea.Markus at jboss.com
+ */
+public abstract class BaseRpcInvokingCommand extends BaseRpcCommand {
+
+   protected InterceptorChain interceptorChain;
+   protected InvocationContextContainer icc;
+
+   private static final Log log = LogFactory.getLog(BaseRpcInvokingCommand.class);
+   private static final boolean trace = log.isTraceEnabled();
+
+   protected BaseRpcInvokingCommand(String cacheName) {
+      super(cacheName);
+   }
+
+   BaseRpcInvokingCommand() {
+   }
+
+   public void init(InterceptorChain interceptorChain, InvocationContextContainer icc) {
+      this.interceptorChain = interceptorChain;
+      this.icc = icc;
+   }
+
+   protected final Object processVisitableCommand(ReplicableCommand cacheCommand) throws Throwable {
+      if (cacheCommand instanceof VisitableCommand) {
+         InvocationContext ctx = icc.createRemoteInvocationContext();
+         if (trace)
+            log.trace("Invoking command " + cacheCommand + ", with originLocal flag set to " + ctx.isOriginLocal() + ".");
+         return interceptorChain.invoke(ctx, (VisitableCommand) cacheCommand);
+         // we only need to return values for a set of remote calls; not every call.
+      } else {
+         throw new RuntimeException("Do we still need to deal with non-visitable commands? (" + cacheCommand.getClass().getName() + ")");
+      }
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/commands/remote/BaseRpcInvokingCommand.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Modified: trunk/core/src/main/java/org/infinispan/commands/remote/ClusteredGetCommand.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/remote/ClusteredGetCommand.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/commands/remote/ClusteredGetCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -27,11 +27,12 @@
 import org.infinispan.container.entries.CacheEntry;
 import org.infinispan.container.entries.InternalCacheEntry;
 import org.infinispan.container.entries.InternalCacheValue;
+import org.infinispan.container.entries.InternalEntryFactory;
 import org.infinispan.container.entries.MVCCEntry;
-import org.infinispan.container.entries.InternalEntryFactory;
 import org.infinispan.context.InvocationContext;
 import org.infinispan.context.InvocationContextContainer;
 import org.infinispan.context.impl.NonTxInvocationContext;
+import org.infinispan.distribution.DistributionManager;
 import org.infinispan.interceptors.InterceptorChain;
 import org.infinispan.marshall.Ids;
 import org.infinispan.marshall.Marshallable;
@@ -62,6 +63,8 @@
    private CommandsFactory commandsFactory;
    private InterceptorChain invoker;
 
+   private DistributionManager distributionManager;
+
    public ClusteredGetCommand() {
    }
 
@@ -84,6 +87,8 @@
     * @return returns an <code>CacheEntry</code> or null, if no entry is found.
     */
    public InternalCacheValue perform(InvocationContext context) throws Throwable {
+      if (distributionManager != null && distributionManager.isAffectedByRehash(key)) return null;
+
       GetKeyValueCommand command = commandsFactory.buildGetKeyValueCommand(key);
       command.setReturnCacheEntry(true);
       NonTxInvocationContext invocationContext = icc.createRemoteInvocationContext();
@@ -144,4 +149,8 @@
    public String getCacheName() {
       return cacheName;
    }
+
+   public Object getKey() {
+      return key;
+   }
 }

Modified: trunk/core/src/main/java/org/infinispan/commands/remote/MultipleRpcCommand.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/remote/MultipleRpcCommand.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/commands/remote/MultipleRpcCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -45,7 +45,7 @@
  * @since 4.0
  */
 @Marshallable(externalizer = ReplicableCommandExternalizer.class, id = Ids.MULTIPLE_RPC_COMMAND)
-public class MultipleRpcCommand extends BaseRpcCommand {
+public class MultipleRpcCommand extends BaseRpcInvokingCommand {
 
    public static final byte COMMAND_ID = 2;
 
@@ -127,7 +127,6 @@
    @Override
    public String toString() {
       return "MultipleRpcCommand{" +
-            "interceptorChain=" + interceptorChain +
             ", commands=" + (commands == null ? null : Arrays.asList(commands)) +
             ", cacheName='" + cacheName + '\'' +
             '}';

Modified: trunk/core/src/main/java/org/infinispan/commands/remote/SingleRpcCommand.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/commands/remote/SingleRpcCommand.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/commands/remote/SingleRpcCommand.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -15,7 +15,7 @@
  * @author Mircea.Markus at jboss.com
  */
 @Marshallable(externalizer = ReplicableCommandExternalizer.class, id = Ids.SINGLE_RPC_COMMAND)
-public class SingleRpcCommand extends BaseRpcCommand {
+public class SingleRpcCommand extends BaseRpcInvokingCommand {
    public static final int COMMAND_ID = 1;
    private static Log log = LogFactory.getLog(SingleRpcCommand.class);
    private static boolean trace = log.isTraceEnabled();
@@ -74,7 +74,6 @@
    @Override
    public String toString() {
       return "SingleRpcCommand{" +
-            "interceptorChain=" + interceptorChain +
             ", cacheName='" + cacheName + '\'' +
             ", command=" + command +
             '}';

Added: trunk/core/src/main/java/org/infinispan/distribution/AbstractConsistentHash.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/AbstractConsistentHash.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/distribution/AbstractConsistentHash.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,22 @@
+package org.infinispan.distribution;
+
+import org.infinispan.remoting.transport.Address;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+public abstract class AbstractConsistentHash implements ConsistentHash {
+   public Map<Object, List<Address>> locateAll(Collection<Object> keys, int replCount) {
+      Map<Object, List<Address>> locations = new HashMap<Object, List<Address>>();
+      for (Object k : keys) locations.put(k, locate(k, replCount));
+      return locations;
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/distribution/AbstractConsistentHash.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Modified: trunk/core/src/main/java/org/infinispan/distribution/ConsistentHash.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/ConsistentHash.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/distribution/ConsistentHash.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -16,8 +16,9 @@
 
    void setCaches(Collection<Address> caches);
 
+   Collection<Address> getCaches();
+
    List<Address> locate(Object key, int replCount);
 
    Map<Object, List<Address>> locateAll(Collection<Object> keys, int replCount);
-
 }

Modified: trunk/core/src/main/java/org/infinispan/distribution/DefaultConsistentHash.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/DefaultConsistentHash.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/distribution/DefaultConsistentHash.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,50 +1,140 @@
 package org.infinispan.distribution;
 
+import org.infinispan.marshall.Ids;
+import org.infinispan.marshall.Marshallable;
 import org.infinispan.remoting.transport.Address;
 
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
 
-public class DefaultConsistentHash implements ConsistentHash {
+ at Marshallable(externalizer = DefaultConsistentHash.Externalizer.class, id = Ids.DEFAULT_CONSISTENT_HASH)
+public class DefaultConsistentHash extends AbstractConsistentHash {
+
    // make sure all threads see the current list
-   volatile ArrayList<Address> addresses;
+   ArrayList<Address> addresses;
+   SortedMap<Integer, Address> positions;
 
+   final static int HASH_SPACE = 10240; // no more than 10k nodes?
+
+
    public void setCaches(Collection<Address> caches) {
+
       addresses = new ArrayList<Address>(caches);
 
       // this list won't grow.
       addresses.trimToSize();
+
+      positions = new TreeMap<Integer, Address>();
+
+      for (Address a : addresses) {
+         int positionIndex = Math.abs(a.hashCode()) % HASH_SPACE;
+         // this is deterministic since the address list is ordered and the order is consistent across the grid
+         while (positions.containsKey(positionIndex)) positionIndex = positionIndex + 1 % HASH_SPACE;
+         positions.put(positionIndex, a);
+      }
    }
 
-   public List<Address> locate(Object key, int replicationCount) {
+   public Collection<Address> getCaches() {
+      return addresses;
+   }
+
+   public List<Address> locate(Object key, int replCount) {
       int hash = Math.abs(key.hashCode());
       int clusterSize = addresses.size();
-      int numCopiesToFind = Math.min(replicationCount, clusterSize);
+      int numCopiesToFind = Math.min(replCount, clusterSize);
 
-      List<Address> results = new ArrayList<Address>(numCopiesToFind);
+      List<Address> owners = new ArrayList<Address>(numCopiesToFind);
 
-      int copyNumber = 0;
+      SortedMap<Integer, Address> candidates = positions.tailMap(hash % HASH_SPACE);
 
-      while (results.size() < numCopiesToFind) {
-         // we mod the index the 2nd time to make sure the index starts again from the beginning when it reaches the end.
-         // e.g., in a cluster of 10 with 3 copies of data, and a key that maps to node index 9, the next 2 backups should
-         // be at indexes 0 and 1.
+      for (Address a : candidates.values()) {
+         if (owners.size() < numCopiesToFind)
+            owners.add(a);
+         else
+            break;
+      }
 
-         int index = ((hash % clusterSize) + copyNumber) % clusterSize;
-         Address candidate = addresses.get(index);
-         results.add(candidate);
-         copyNumber++;
+      if (owners.size() < numCopiesToFind) {
+         for (Address a : positions.values()) {
+            if (owners.size() < numCopiesToFind)
+               owners.add(a);
+            else
+               break;
+         }
       }
 
-      return results;
+      return owners;
    }
 
-   public Map<Object, List<Address>> locateAll(Collection<Object> keys, int replCount) {
-      Map<Object, List<Address>> locations = new HashMap<Object, List<Address>>();
-      for (Object k : keys) locations.put(k, locate(k, replCount));
-      return locations;
+//   public List<Address> locate(Object key, int replicationCount) {
+//      int hash = Math.abs(key.hashCode());
+//      int clusterSize = addresses.size();
+//      int numCopiesToFind = Math.min(replicationCount, clusterSize);
+//
+//      List<Address> results = new ArrayList<Address>(numCopiesToFind);
+//
+//      int copyNumber = 0;
+//
+//      while (results.size() < numCopiesToFind) {
+//         // we mod the index the 2nd time to make sure the index starts again from the beginning when it reaches the end.
+//         // e.g., in a cluster of 10 with 3 copies of data, and a key that maps to node index 9, the next 2 backups should
+//         // be at indexes 0 and 1.
+//
+//         int index = ((hash % clusterSize) + copyNumber) % clusterSize;
+//         Address candidate = addresses.get(index);
+//         results.add(candidate);
+//         copyNumber++;
+//      }
+//
+//      return results;
+//   }
+
+   @Override
+   public boolean equals(Object o) {
+      if (this == o) return true;
+      if (o == null || getClass() != o.getClass()) return false;
+
+      DefaultConsistentHash that = (DefaultConsistentHash) o;
+
+      if (addresses != null ? !addresses.equals(that.addresses) : that.addresses != null) return false;
+      if (positions != null ? !positions.equals(that.positions) : that.positions != null) return false;
+
+      return true;
    }
+
+   @Override
+   public int hashCode() {
+      int result = addresses != null ? addresses.hashCode() : 0;
+      result = 31 * result + (positions != null ? positions.hashCode() : 0);
+      return result;
+   }
+
+   public static class Externalizer implements org.infinispan.marshall.Externalizer {
+      public void writeObject(ObjectOutput output, Object subject) throws IOException {
+         DefaultConsistentHash dch = (DefaultConsistentHash) subject;
+         output.writeObject(dch.addresses);
+         output.writeObject(dch.positions);
+      }
+
+      @SuppressWarnings("unchecked")
+      public Object readObject(ObjectInput unmarshaller) throws IOException, ClassNotFoundException {
+         DefaultConsistentHash dch = new DefaultConsistentHash();
+         dch.addresses = (ArrayList<Address>) unmarshaller.readObject();
+         dch.positions = (SortedMap<Integer, Address>) unmarshaller.readObject();
+         return dch;
+      }
+   }
+
+   @Override
+   public String toString() {
+      return "DefaultConsistentHash{" +
+            "addresses=" + addresses +
+            '}';
+   }
 }

Modified: trunk/core/src/main/java/org/infinispan/distribution/DistributionManager.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/DistributionManager.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/distribution/DistributionManager.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -19,8 +19,6 @@
 @Scope(Scopes.NAMED_CACHE)
 public interface DistributionManager {
 
-   void rehash(Collection<Address> newList);
-
    boolean isLocal(Object key);
 
    /**
@@ -57,5 +55,43 @@
     * @return an internal cache entry, or null if it cannot be located
     */
    InternalCacheEntry retrieveFromRemoteSource(Object key) throws Exception;
+
+   ConsistentHash getConsistentHash();
+
+   void setConsistentHash(ConsistentHash consistentHash);
+
+   /**
+    * Tests whether a given key is affected by a rehash that may be in progress.
+    *
+    * @param key key to test
+    * @return whether a key is affected by a rehash
+    */
+   boolean isAffectedByRehash(Object key);
+
+   TransactionLogger getTransactionLogger();
+
+   /**
+    * "Asks" a coordinator if a joiner may join.  Used to serialize joins such that only a single joiner comes in at any
+    * given time.
+    *
+    * @param joiner joiner who wants to join
+    * @return a consistent hash prior to the joiner joining (if the joiner is allowed to join), otherwise null.
+    */
+   List<Address> requestPermissionToJoin(Address joiner);
+
+   /**
+    * Notifies a coordinator when a join completes
+    *
+    * @param joiner joiner who has completed.
+    */
+   void notifyJoinComplete(Address joiner);
+
+   /**
+    * This will cause all nodes to add the joiner to their UnionCH
+    *
+    * @param joiner
+    * @param starting
+    */
+   void informRehashOnJoin(Address joiner, boolean starting);
 }
 

Modified: trunk/core/src/main/java/org/infinispan/distribution/DistributionManagerImpl.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/DistributionManagerImpl.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/distribution/DistributionManagerImpl.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,14 +1,18 @@
 package org.infinispan.distribution;
 
+import org.infinispan.CacheException;
 import org.infinispan.commands.CommandsFactory;
 import org.infinispan.commands.remote.ClusteredGetCommand;
 import org.infinispan.config.Configuration;
+import org.infinispan.container.DataContainer;
 import org.infinispan.container.entries.CacheEntry;
 import org.infinispan.container.entries.InternalCacheEntry;
 import org.infinispan.container.entries.InternalCacheValue;
+import org.infinispan.context.InvocationContextContainer;
 import org.infinispan.factories.annotations.Inject;
 import org.infinispan.factories.annotations.Start;
 import org.infinispan.factories.annotations.Stop;
+import org.infinispan.interceptors.InterceptorChain;
 import org.infinispan.notifications.Listener;
 import org.infinispan.notifications.cachemanagerlistener.CacheManagerNotifier;
 import org.infinispan.notifications.cachemanagerlistener.annotation.ViewChanged;
@@ -24,9 +28,18 @@
 import org.infinispan.util.logging.Log;
 import org.infinispan.util.logging.LogFactory;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.concurrent.locks.LockSupport;
 
 /**
  * The default distribution manager implementation
@@ -44,13 +57,40 @@
    int replCount;
    ViewChangeListener listener;
    CommandsFactory cf;
+   LinkedBlockingQueue<Runnable> rehashQueue = new LinkedBlockingQueue<Runnable>();
+   ThreadFactory tf = new ThreadFactory() {
+      public Thread newThread(Runnable r) {
+         Thread t = new Thread(r);
+         t.setDaemon(true);
+         t.setPriority(Thread.MIN_PRIORITY);
+         t.setName("Rehasher-" + rpcManager.getTransport().getAddress());
+         return t;
+      }
+   };
+   ExecutorService rehashExecutor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, rehashQueue, tf);
 
+   TransactionLogger transactionLogger = new TransactionLoggerImpl();
+   volatile boolean rehashInProgress = false;
+   volatile Address joiner;
+   static final AtomicReferenceFieldUpdater<DistributionManagerImpl, Address> JOINER_CAS =
+         AtomicReferenceFieldUpdater.newUpdater(DistributionManagerImpl.class, Address.class, "joiner");
+   private DataContainer dataContainer;
+   private InterceptorChain interceptorChain;
+   private InvocationContextContainer icc;
+   private volatile boolean joinTaskSubmitted = false;
+   volatile boolean joinComplete = false;
+
+
    @Inject
-   public void init(Configuration configuration, RpcManager rpcManager, CacheManagerNotifier notifier, CommandsFactory cf) {
+   public void init(Configuration configuration, RpcManager rpcManager, CacheManagerNotifier notifier, CommandsFactory cf,
+                    DataContainer dataContainer, InterceptorChain interceptorChain, InvocationContextContainer icc) {
       this.configuration = configuration;
       this.rpcManager = rpcManager;
       this.notifier = notifier;
       this.cf = cf;
+      this.dataContainer = dataContainer;
+      this.interceptorChain = interceptorChain;
+      this.icc = icc;
    }
 
    // needs to be AFTER the RpcManager
@@ -61,16 +101,47 @@
       consistentHash.setCaches(rpcManager.getTransport().getMembers());
       listener = new ViewChangeListener();
       notifier.addListener(listener);
+      if (rpcManager.getTransport().getMembers().size() > 1) {
+         JoinTask joinTask = new JoinTask(rpcManager, cf, configuration, transactionLogger, dataContainer, interceptorChain, icc, this);
+         rehashExecutor.submit(joinTask);
+      } else {
+         joinComplete = true;
+      }
+      joinTaskSubmitted = true;
    }
 
    @Stop(priority = 20)
    public void stop() {
       notifier.removeListener(listener);
+      rehashExecutor.shutdownNow();
+      joinComplete = false;
    }
 
-   public void rehash(Collection<Address> newList) {
+   private Address diff(List<Address> newList, List<Address> oldList) {
+      List<Address> list = new ArrayList<Address>(newList);
+      list.removeAll(oldList);
+      // Could easily be > 1 member joined!
+      return list.size() > 0 ? list.get(0) : null;
+   }
+
+
+   public void rehash(List<Address> newMembers, List<Address> oldMembers) {
+      boolean join = oldMembers == null || oldMembers.size() < newMembers.size();
       // on view change, we should update our view
-      consistentHash.setCaches(newList);
+      log.info("Detected a veiw change.  Member list changed from {0} to {1}", oldMembers, newMembers);
+
+      if (join) {
+         Address joiner = diff(newMembers, oldMembers);
+         log.info("This is a JOIN event!  Wait for notification from new joiner " + joiner);
+      } else {
+         log.info("This is a LEAVE event!");
+         // TODO: implement this stuff!
+         ConsistentHash newCH = new DefaultConsistentHash();
+         newCH.setCaches(newMembers);
+
+         consistentHash = newCH;
+
+      }
    }
 
    public boolean isLocal(Object key) {
@@ -109,11 +180,75 @@
       return null;
    }
 
+   public ConsistentHash getConsistentHash() {
+      return consistentHash;
+   }
+
+   public void setConsistentHash(ConsistentHash consistentHash) {
+      log.trace("Installing new consistent hash {0}", consistentHash);
+      this.consistentHash = consistentHash;
+   }
+
+   public boolean isAffectedByRehash(Object key) {
+      if (rehashInProgress) {
+         throw new UnsupportedOperationException("TODO implement me");
+      } else {
+         return false;
+      }
+   }
+
+   public TransactionLogger getTransactionLogger() {
+      return transactionLogger;
+   }
+
+   public List<Address> requestPermissionToJoin(Address joiner) {
+      if (JOINER_CAS.compareAndSet(this, null, joiner))
+         return new LinkedList<Address>(consistentHash.getCaches());
+      else
+         return null;
+   }
+
+   public void notifyJoinComplete(Address joiner) {
+      log.trace("Received notification that {0} has completed a join.  Current 'joiner' flag is {1}, setting this to null.", joiner, this.joiner);
+      if (this.joiner != null) {
+         if (this.joiner.equals(joiner)) this.joiner = null;
+      }
+   }
+
+   public void informRehashOnJoin(Address joiner, boolean starting) {
+      log.trace("Informed of a JOIN by {0}.  Starting? {1}", joiner, starting);
+      if (!starting) {
+         if (consistentHash instanceof UnionConsistentHash) {
+            UnionConsistentHash uch = (UnionConsistentHash) consistentHash;
+            consistentHash = uch.getNewConsistentHash();
+         }
+         rehashInProgress = false;
+      } else {
+         ConsistentHash chOld = consistentHash;
+         if (chOld instanceof UnionConsistentHash) throw new RuntimeException("Not expecting a union CH!");
+         this.joiner = joiner;
+         rehashInProgress = true;
+
+         ConsistentHash chNew;
+         try {
+            chNew = (ConsistentHash) Util.getInstance(configuration.getConsistentHashClass());
+         } catch (Exception e) {
+            throw new CacheException("Unable to create instance of " + configuration.getConsistentHashClass(), e);
+         }
+         List<Address> newAddresses = new LinkedList<Address>(chOld.getCaches());
+         newAddresses.add(joiner);
+         chNew.setCaches(newAddresses);
+         consistentHash = new UnionConsistentHash(chOld, chNew);
+      }
+      log.trace("New CH is {0}", consistentHash);
+   }
+
    @Listener
    public class ViewChangeListener {
       @ViewChanged
       public void handleViewChange(ViewChangedEvent e) {
-         rehash(e.getNewMemberList());
+         while (!joinTaskSubmitted) LockSupport.parkNanos(100 * 1000000);
+         rehash(e.getNewMembers(), e.getOldMembers());
       }
    }
 }

Added: trunk/core/src/main/java/org/infinispan/distribution/JoinTask.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/JoinTask.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/distribution/JoinTask.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,214 @@
+package org.infinispan.distribution;
+
+import org.infinispan.CacheException;
+import org.infinispan.commands.CommandsFactory;
+import org.infinispan.commands.control.PullStateCommand;
+import org.infinispan.commands.write.InvalidateCommand;
+import org.infinispan.commands.write.PutKeyValueCommand;
+import org.infinispan.commands.write.WriteCommand;
+import org.infinispan.config.Configuration;
+import org.infinispan.container.DataContainer;
+import org.infinispan.container.entries.InternalCacheValue;
+import org.infinispan.context.Flag;
+import org.infinispan.context.InvocationContext;
+import org.infinispan.context.InvocationContextContainer;
+import org.infinispan.interceptors.InterceptorChain;
+import org.infinispan.remoting.responses.Response;
+import org.infinispan.remoting.responses.SuccessfulResponse;
+import org.infinispan.remoting.rpc.ResponseMode;
+import org.infinispan.remoting.rpc.RpcManager;
+import org.infinispan.remoting.transport.Address;
+import org.infinispan.util.Util;
+import org.infinispan.util.concurrent.NotifyingFutureImpl;
+import org.infinispan.util.concurrent.NotifyingNotifiableFuture;
+import org.infinispan.util.logging.Log;
+import org.infinispan.util.logging.LogFactory;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Future;
+
+/**
+ * 5.  JoinTask: This is a PULL based rehash.  JoinTask is kicked off on the JOINER. 5.1.  Obtain OLD_CH from
+ * coordinator (using GetConsistentHashCommand) 5.2.  Generate TEMP_CH (which is a union of OLD_CH and NEW_CH) 5.3.
+ * Broadcast TEMP_CH across the cluster (using InstallConsistentHashCommand) 5.4.  Log all incoming writes/txs and
+ * respond with a positive ack. 5.5.  Ignore incoming reads, forcing callers to check next owner of data. 5.6.  Ping
+ * each node in OLD_CH's view and ask for state (PullStateCommand) 5.7.  Apply state received from 5.6. 5.8.  Drain tx
+ * log and apply, stop logging writes once drained. 5.9.  Reverse 5.5. 5.10. Broadcast NEW_CH so this is applied (using
+ * InstallConsistentHashCommand) 5.11. Loop through data container and unicast invalidations for keys that "could" exist
+ * on OLD_CH and not in NEW_CH
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+public class JoinTask extends RehashTask {
+
+   private static final Log log = LogFactory.getLog(JoinTask.class);
+   ConsistentHash chOld;
+   ConsistentHash chNew;
+   //   ConsistentHash chTemp;
+   CommandsFactory commandsFactory;
+   TransactionLogger transactionLogger;
+   DataContainer dataContainer;
+   InterceptorChain interceptorChain;
+   InvocationContextContainer icc;
+   Address self;
+
+   public JoinTask(RpcManager rpcManager, CommandsFactory commandsFactory, Configuration conf,
+                   TransactionLogger transactionLogger, DataContainer dataContainer, InterceptorChain interceptorChain,
+                   InvocationContextContainer icc, DistributionManagerImpl dmi) {
+      super(dmi, rpcManager, conf);
+      this.commandsFactory = commandsFactory;
+      this.transactionLogger = transactionLogger;
+      this.dataContainer = dataContainer;
+      this.interceptorChain = interceptorChain;
+      this.icc = icc;
+      self = rpcManager.getTransport().getAddress();
+   }
+
+   protected void performRehash() throws Exception {
+      log.trace("Starting rehash on new joiner");
+      boolean unlocked = false;
+      try {
+         dmi.joinComplete = false;
+         // 1.  Get chOld from coord.
+         // this happens in a loop to ensure we receive the correct CH and not a "union".
+         // TODO make at least *some* of these configurable!
+         long sleepTime = 500; // sleep time between retries
+         int incrementFactor = 2; // factor by wich to increment retry sleep
+         int maxSleepTime = 600000; // after which we give up!
+
+         do {
+            log.trace("Requesting old consistent hash from coordinator");
+            List<Response> resp = rpcManager.invokeRemotely(coordinator(), commandsFactory.buildGetConsistentHashCommand(self),
+                                                            ResponseMode.SYNCHRONOUS, 100000, true);
+            for (Response r : resp) {
+               if (r instanceof SuccessfulResponse) {
+                  List<Address> list = (List<Address>) ((SuccessfulResponse) r).getResponseValue();
+                  chOld = createConsistentHash(list);
+                  break;
+               }
+            }
+
+            log.trace("Retrieved old consistent hash {0}", chOld);
+            if (chOld == null) {
+               if (sleepTime > maxSleepTime)
+                  throw new CacheException("Unable to retrieve old consistent hash from coordinator even after several attempts at sleeping and retrying!");
+               log.debug("Sleeping for {0}", Util.prettyPrintTime(sleepTime));
+               Thread.sleep(sleepTime); // sleep for a while and retry
+               sleepTime *= incrementFactor;
+            }
+         } while (chOld == null);
+
+         // 2.  new CH instance
+         chNew = createConsistentHash(chOld.getCaches(), self);
+
+         // 3.  Enable TX logging
+         transactionLogger.enable();
+
+         // 4.  Broadcast new temp CH
+         rpcManager.broadcastRpcCommand(commandsFactory.buildInstallConsistentHashCommand(self, true), true, true);
+
+         // 5.  txLogger being enabled will cause CLusteredGetCommands to return uncertain responses.
+
+         // 6.  pull state from everyone.
+         Address myAddress = rpcManager.getTransport().getAddress();
+         PullStateCommand cmd = commandsFactory.buildPullStateCommand(myAddress, chNew);
+         // TODO I should be able to process state chunks from different nodes simultaneously!!
+         // TODO I should only send this pull state request to nodes which I know will send me state.  Not everyone in chOld!!
+         List<Response> resps = rpcManager.invokeRemotely(chOld.getCaches(), cmd, ResponseMode.SYNCHRONOUS, 10000, true);
+
+         // 7.  Apply state
+         for (Response r : resps) {
+            if (r instanceof SuccessfulResponse) {
+               Map<Object, InternalCacheValue> state = (Map<Object, InternalCacheValue>) ((SuccessfulResponse) r).getResponseValue();
+               for (Map.Entry<Object, InternalCacheValue> e : state.entrySet()) {
+                  if (chNew.locate(e.getKey(), configuration.getNumOwners()).contains(myAddress)) {
+                     InternalCacheValue v = e.getValue();
+                     PutKeyValueCommand put = commandsFactory.buildPutKeyValueCommand(e.getKey(), v.getValue(), v.getLifespan(), v.getMaxIdle());
+                     InvocationContext ctx = icc.createInvocationContext();
+                     ctx.setFlags(Flag.CACHE_MODE_LOCAL, Flag.SKIP_REMOTE_LOOKUP);
+                     interceptorChain.invoke(ctx, put);
+                  }
+               }
+            }
+         }
+
+         // 8.  Drain logs
+
+         List<WriteCommand> c = null;
+         while (transactionLogger.size() > 10) {
+            c = transactionLogger.drain();
+            apply(c);
+         }
+
+         c = transactionLogger.drainAndLock();
+         apply(c);
+
+         unlocked = true;
+         // 9.
+         transactionLogger.unlockAndDisable();
+
+         // 10.
+         // TODO this phase should also "tell" the coord that the join is complete so that other waiting joiners
+         // may proceed.  Ideally another command, directed to the coord.
+         rpcManager.broadcastRpcCommand(commandsFactory.buildInstallConsistentHashCommand(self, false), true, true);
+         rpcManager.invokeRemotely(coordinator(), commandsFactory.buildJoinCompleteCommand(self), ResponseMode.SYNCHRONOUS,
+                                   100000, true);
+
+         // 11.
+         Map<Address, Set<Object>> invalidations = new HashMap<Address, Set<Object>>();
+         for (Object key : dataContainer.keySet()) {
+            Collection<Address> invalidHolders = getInvalidHolders(key, chOld, chNew);
+            for (Address a : invalidHolders) {
+               Set<Object> s = invalidations.get(a);
+               if (s == null) {
+                  s = new HashSet<Object>();
+                  invalidations.put(a, s);
+               }
+               s.add(key);
+            }
+         }
+
+         Set<Future> futures = new HashSet<Future>();
+
+         for (Map.Entry<Address, Set<Object>> e : invalidations.entrySet()) {
+            InvalidateCommand ic = commandsFactory.buildInvalidateFromL1Command(e.getValue().toArray());
+            NotifyingNotifiableFuture f = new NotifyingFutureImpl(null);
+            rpcManager.invokeRemotelyInFuture(Collections.singletonList(e.getKey()), ic, true, f);
+            futures.add(f);
+         }
+
+         for (Future f : futures) f.get();
+      } catch (Exception e) {
+         log.warn("Caught error performing rehash!", e);
+      } finally {
+         if (!unlocked) transactionLogger.unlockAndDisable();
+         dmi.joinComplete = true;
+      }
+   }
+
+   private Collection<Address> getInvalidHolders(Object key, ConsistentHash chOld, ConsistentHash chNew) {
+      List<Address> oldOwners = chOld.locate(key, configuration.getNumOwners());
+      List<Address> newOwners = chNew.locate(key, configuration.getNumOwners());
+
+      List<Address> toInvalidate = new LinkedList<Address>(oldOwners);
+      toInvalidate.removeAll(newOwners);
+
+      return toInvalidate;
+   }
+
+   private void apply(List<WriteCommand> c) {
+      for (WriteCommand cmd : c) {
+         InvocationContext ctx = icc.createInvocationContext();
+         ctx.setFlags(Flag.SKIP_REMOTE_LOOKUP);
+         interceptorChain.invoke(ctx, cmd);
+      }
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/distribution/JoinTask.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/distribution/LeaveTask.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/LeaveTask.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/distribution/LeaveTask.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,20 @@
+package org.infinispan.distribution;
+
+import org.infinispan.config.Configuration;
+import org.infinispan.remoting.rpc.RpcManager;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+public class LeaveTask extends RehashTask {
+   protected LeaveTask(DistributionManagerImpl dmi, RpcManager rpcManager, Configuration configuration) {
+      super(dmi, rpcManager, configuration);
+   }
+
+   protected void performRehash() throws Exception {
+      // TODO: Customise this generated block
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/distribution/LeaveTask.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/distribution/RecvLeaveTask.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/RecvLeaveTask.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/distribution/RecvLeaveTask.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,16 @@
+package org.infinispan.distribution;
+
+import org.infinispan.config.Configuration;
+import org.infinispan.remoting.rpc.RpcManager;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+public class RecvLeaveTask extends LeaveTask {
+   protected RecvLeaveTask(DistributionManagerImpl dmi, RpcManager rpcManager, Configuration configuration) {
+      super(dmi, rpcManager, configuration);
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/distribution/RecvLeaveTask.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/distribution/RehashHandler.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/RehashHandler.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/distribution/RehashHandler.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,18 @@
+package org.infinispan.distribution;
+
+import org.infinispan.commands.CommandsFactory;
+import org.infinispan.container.DataContainer;
+import org.infinispan.loaders.CacheStore;
+import org.infinispan.remoting.rpc.RpcManager;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+public interface RehashHandler {
+
+   void rehash(DataContainer dc, CacheStore cacheStore, RpcManager rpc, CommandsFactory cf);
+
+}


Property changes on: trunk/core/src/main/java/org/infinispan/distribution/RehashHandler.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/distribution/RehashTask.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/RehashTask.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/distribution/RehashTask.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,60 @@
+package org.infinispan.distribution;
+
+import org.infinispan.config.Configuration;
+import org.infinispan.remoting.rpc.RpcManager;
+import org.infinispan.remoting.transport.Address;
+import org.infinispan.util.Util;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+public abstract class RehashTask implements Callable<Void> {
+
+   DistributionManagerImpl dmi;
+   RpcManager rpcManager;
+   Configuration configuration;
+
+   protected RehashTask(DistributionManagerImpl dmi, RpcManager rpcManager, Configuration configuration) {
+      this.dmi = dmi;
+      this.rpcManager = rpcManager;
+      this.configuration = configuration;
+   }
+
+   public Void call() throws Exception {
+      dmi.rehashInProgress = true;
+      try {
+         performRehash();
+         return null;
+      } finally {
+         dmi.rehashInProgress = false;
+      }
+   }
+
+   protected abstract void performRehash() throws Exception;
+
+   protected Collection<Address> coordinator() {
+      return Collections.singleton(rpcManager.getTransport().getCoordinator());
+   }
+
+   protected ConsistentHash createConsistentHash(Collection<Address> addresses) throws Exception {
+      ConsistentHash ch = (ConsistentHash) Util.getInstance(configuration.getConsistentHashClass());
+      ch.setCaches(addresses);
+      return ch;
+   }
+
+   protected ConsistentHash createConsistentHash(Collection<Address> addresses, Address... moreAddresses) throws Exception {
+      List<Address> list = new LinkedList<Address>(addresses);
+      list.addAll(Arrays.asList(moreAddresses));
+      return createConsistentHash(list);
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/distribution/RehashTask.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/distribution/SendLeaveTask.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/SendLeaveTask.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/distribution/SendLeaveTask.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,16 @@
+package org.infinispan.distribution;
+
+import org.infinispan.config.Configuration;
+import org.infinispan.remoting.rpc.RpcManager;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+public class SendLeaveTask extends LeaveTask {
+   protected SendLeaveTask(DistributionManagerImpl dmi, RpcManager rpcManager, Configuration configuration) {
+      super(dmi, rpcManager, configuration);
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/distribution/SendLeaveTask.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/distribution/TransactionLogger.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/TransactionLogger.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/distribution/TransactionLogger.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,59 @@
+package org.infinispan.distribution;
+
+import org.infinispan.commands.write.WriteCommand;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Typically adding a command, the following pattern would be used:
+ * <p/>
+ * <code> if (txLogger.logIfNeeded(cmd)) { // do NOT proceed with executing this command! } else { // proceed with
+ * executing this command as per normal! } </code>
+ * <p/>
+ * When draining, the following pattern should be used:
+ * <p/>
+ * <code> List<WriteCommand> c = null; while (txLogger.size() > THRESHOLD) { c = txLogger.drain(); applyCommands(c); } c
+ * = txLogger.drainAndLock(); applyCommands(c); txLogger.unlockAndDisable(); </code>
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+public interface TransactionLogger {
+   void enable();
+
+   /**
+    * Drains the transaction log and returns a list of what has been drained.
+    *
+    * @return a list of drained commands
+    */
+   List<WriteCommand> drain();
+
+   /**
+    * Similar to {@link #drain()} except that relevant locks are acquired so that no more commands are added to the
+    * transaction log during this process, and transaction logging is disabled after draining.
+    *
+    * @return list of drained commands
+    */
+   List<WriteCommand> drainAndLock();
+
+   void unlockAndDisable();
+
+   /**
+    * If logging is enabled, will log the command and return true.  Otherwise, will just return false.
+    *
+    * @param command command to log
+    * @return true if logged, false otherwise
+    */
+   boolean logIfNeeded(WriteCommand command);
+
+   /**
+    * If logging is enabled, will log the commands and return true.  Otherwise, will just return false.
+    *
+    * @param commands commands to log
+    * @return true if logged, false otherwise
+    */
+   boolean logIfNeeded(Collection<WriteCommand> commands);
+
+   int size();
+}


Property changes on: trunk/core/src/main/java/org/infinispan/distribution/TransactionLogger.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/distribution/TransactionLoggerImpl.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/TransactionLoggerImpl.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/distribution/TransactionLoggerImpl.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,75 @@
+package org.infinispan.distribution;
+
+import org.infinispan.commands.write.WriteCommand;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+public class TransactionLoggerImpl implements TransactionLogger {
+   volatile boolean enabled;
+   final ReadWriteLock loggingLock = new ReentrantReadWriteLock();
+   final BlockingQueue<WriteCommand> commandQueue = new LinkedBlockingQueue<WriteCommand>();
+
+   public void enable() {
+      enabled = true;
+   }
+
+   public List<WriteCommand> drain() {
+      List<WriteCommand> list = new LinkedList<WriteCommand>();
+      commandQueue.drainTo(list);
+      return list;
+   }
+
+   public List<WriteCommand> drainAndLock() {
+      loggingLock.writeLock().lock();
+      return drain();
+   }
+
+   public void unlockAndDisable() {
+      enabled = false;
+      loggingLock.writeLock().unlock();
+   }
+
+   public boolean logIfNeeded(WriteCommand command) {
+      loggingLock.readLock().lock();
+      try {
+         if (enabled) {
+            commandQueue.add(command);
+            return true;
+         } else {
+            return false;
+         }
+      } finally {
+         loggingLock.readLock().unlock();
+      }
+   }
+
+   public boolean logIfNeeded(Collection<WriteCommand> commands) {
+      loggingLock.readLock().lock();
+      try {
+         if (enabled) {
+            for (WriteCommand command : commands) commandQueue.add(command);
+            return true;
+         } else {
+            return false;
+         }
+      } finally {
+         loggingLock.readLock().unlock();
+      }
+   }
+
+   public int size() {
+      return enabled ? 0 : commandQueue.size();
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/distribution/TransactionLoggerImpl.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Added: trunk/core/src/main/java/org/infinispan/distribution/UnionConsistentHash.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/distribution/UnionConsistentHash.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/distribution/UnionConsistentHash.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,71 @@
+package org.infinispan.distribution;
+
+import org.infinispan.CacheException;
+import org.infinispan.marshall.Ids;
+import org.infinispan.marshall.Marshallable;
+import org.infinispan.remoting.transport.Address;
+import org.infinispan.util.Immutables;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * // TODO: Manik: Document this
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+ at Marshallable(externalizer = UnionConsistentHash.Externalizer.class, id = Ids.UNION_CONSISTENT_HASH)
+public class UnionConsistentHash extends AbstractConsistentHash {
+
+   ConsistentHash oldCH, newCH;
+
+   public UnionConsistentHash(ConsistentHash oldCH, ConsistentHash newCH) {
+      if ((oldCH instanceof UnionConsistentHash) || (newCH instanceof UnionConsistentHash))
+         throw new CacheException("Expecting both newCH and oldCH to not be Unions!!  oldCH=[" + oldCH.getClass() + "] and newCH=[" + newCH.getClass() + "]");
+      this.oldCH = oldCH;
+      this.newCH = newCH;
+   }
+
+   public void setCaches(Collection<Address> caches) {
+      // no op
+   }
+
+   public Collection<Address> getCaches() {
+      return Collections.emptyList();
+   }
+
+   public List<Address> locate(Object key, int replCount) {
+      Set<Address> addresses = new LinkedHashSet<Address>();
+      addresses.addAll(oldCH.locate(key, replCount));
+      addresses.addAll(newCH.locate(key, replCount));
+      return Immutables.immutableListConvert(addresses);
+   }
+
+   public ConsistentHash getNewConsistentHash() {
+      return newCH;
+   }
+
+   public ConsistentHash getOldConsistentHash() {
+      return oldCH;
+   }
+
+   public static class Externalizer implements org.infinispan.marshall.Externalizer {
+
+      public void writeObject(ObjectOutput output, Object object) throws IOException {
+         UnionConsistentHash uch = (UnionConsistentHash) object;
+         output.writeObject(uch.oldCH);
+         output.writeObject(uch.newCH);
+      }
+
+      public Object readObject(ObjectInput input) throws IOException, ClassNotFoundException {
+         return new UnionConsistentHash((ConsistentHash) input.readObject(), (ConsistentHash) input.readObject());
+      }
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/distribution/UnionConsistentHash.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Modified: trunk/core/src/main/java/org/infinispan/loaders/decorators/SingletonStore.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/loaders/decorators/SingletonStore.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/loaders/decorators/SingletonStore.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,7 +1,6 @@
 package org.infinispan.loaders.decorators;
 
 import org.infinispan.Cache;
-import org.infinispan.transaction.xa.GlobalTransaction;
 import org.infinispan.container.DataContainer;
 import org.infinispan.container.entries.InternalCacheEntry;
 import org.infinispan.loaders.CacheLoaderException;
@@ -14,6 +13,7 @@
 import org.infinispan.notifications.cachemanagerlistener.event.Event;
 import org.infinispan.notifications.cachemanagerlistener.event.ViewChangedEvent;
 import org.infinispan.remoting.transport.Address;
+import org.infinispan.transaction.xa.GlobalTransaction;
 import org.infinispan.util.logging.Log;
 import org.infinispan.util.logging.LogFactory;
 
@@ -319,7 +319,7 @@
        */
       @ViewChanged
       public void viewChange(ViewChangedEvent event) {
-         boolean tmp = isCoordinator(event.getNewMemberList(), event.getLocalAddress());
+         boolean tmp = isCoordinator(event.getNewMembers(), event.getLocalAddress());
 
          if (active != tmp) {
             try {

Modified: trunk/core/src/main/java/org/infinispan/marshall/Ids.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/marshall/Ids.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/marshall/Ids.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -23,35 +23,39 @@
 
 /**
  * Indexes.
- * 
+ *
  * @author Galder Zamarreño
  * @since 4.0
  */
 public interface Ids {
-   /** ids for jdk classes **/
-   
+   /**
+    * ids for jdk classes *
+    */
+
    static final byte ARRAY_LIST = 0;
    static final byte LINKED_LIST = 1;
    static final byte JDK_MAPS = 2;
    static final byte JDK_SETS = 3;
-   static final byte SINGLETON_LIST = 4; 
+   static final byte SINGLETON_LIST = 4;
 
-   /** ids for infinispan core classes **/
+   /**
+    * ids for infinispan core classes *
+    */
 
    // responses
    static final byte SUCCESSFUL_RESPONSE = 5;
-   static final byte EXTENDED_RESPONSE = 6; 
+   static final byte EXTENDED_RESPONSE = 6;
    static final byte EXCEPTION_RESPONSE = 7;
    static final byte UNSUCCESSFUL_RESPONSE = 8;
    static final byte REQUEST_IGNORED_RESPONSE = 9;
-   
+
    // entries and values
-   static final byte IMMORTAL_ENTRY = 10; 
+   static final byte IMMORTAL_ENTRY = 10;
    static final byte MORTAL_ENTRY = 11;
    static final byte TRANSIENT_ENTRY = 12;
    static final byte TRANSIENT_MORTAL_ENTRY = 13;
-   static final byte IMMORTAL_VALUE = 14; 
-   static final byte MORTAL_VALUE = 15; 
+   static final byte IMMORTAL_VALUE = 14;
+   static final byte MORTAL_VALUE = 15;
    static final byte TRANSIENT_VALUE = 16;
    static final byte TRANSIENT_MORTAL_VALUE = 17;
 
@@ -68,10 +72,10 @@
    static final byte GET_KEY_VALUE_COMMAND = 25;
    static final byte PUT_KEY_VALUE_COMMAND = 26;
    static final byte REMOVE_COMMAND = 27;
-   static final byte INVALIDATE_COMMAND = 28; 
+   static final byte INVALIDATE_COMMAND = 28;
    static final byte REPLACE_COMMAND = 29;
    static final byte CLEAR_COMMAND = 30;
-   static final byte PUT_MAP_COMMAND = 31; 
+   static final byte PUT_MAP_COMMAND = 31;
    static final byte PREPARE_COMMAND = 32;
    static final byte COMMIT_COMMAND = 33;
    static final byte ROLLBACK_COMMAND = 34;
@@ -83,16 +87,27 @@
    static final byte GLOBAL_TRANSACTION = 38;
    static final byte JGROUPS_ADDRESS = 39;
    static final byte MARSHALLED_VALUE = 40;
-   static final byte TRANSACTION_LOG_ENTRY = 41; 
+   static final byte TRANSACTION_LOG_ENTRY = 41;
    static final byte BUCKET = 42;
    static final byte DEADLOCK_DETECTING_GLOBAL_TRANSACTION = 43;
+
+   /**
+    * ids for infinispan tree classes *
+    */
+
+   static final byte NODE_KEY = 44;
+   static final byte FQN = 45;
+
    static final byte ATOMIC_HASH_MAP_DELTA = 46;
    static final byte ATOMIC_PUT_OPERATION = 47;
    static final byte ATOMIC_REMOVE_OPERATION = 48;
    static final byte ATOMIC_CLEAR_OPERATION = 49;
-   
-   /** ids for infinispan tree classes **/
 
-   static final byte NODE_KEY = 44;
-   static final byte FQN = 45;
+   static final byte GET_CONSISTENT_HASH_COMMAND = 50;
+   static final byte PUSH_STATE_COMMAND = 51;
+   static final byte PULL_STATE_COMMAND = 52;
+   static final byte INSTALL_CONSISTENT_HASH_COMMAND = 53;
+   static final byte DEFAULT_CONSISTENT_HASH = 54;
+   static final byte UNION_CONSISTENT_HASH = 55;
+   static final byte JOIN_COMPLETE_COMMAND = 56;
 }

Modified: trunk/core/src/main/java/org/infinispan/marshall/jboss/ConstantObjectTable.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/marshall/jboss/ConstantObjectTable.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/marshall/jboss/ConstantObjectTable.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -27,8 +27,13 @@
 import org.infinispan.atomic.ClearOperation;
 import org.infinispan.atomic.PutOperation;
 import org.infinispan.atomic.RemoveOperation;
+import org.infinispan.commands.RemoteCommandFactory;
+import org.infinispan.commands.control.GetConsistentHashCommand;
+import org.infinispan.commands.control.InstallConsistentHashCommand;
+import org.infinispan.commands.control.JoinCompleteCommand;
 import org.infinispan.commands.control.LockControlCommand;
-import org.infinispan.commands.RemoteCommandFactory;
+import org.infinispan.commands.control.PullStateCommand;
+import org.infinispan.commands.control.PushStateCommand;
 import org.infinispan.commands.control.StateTransferControlCommand;
 import org.infinispan.commands.read.GetKeyValueCommand;
 import org.infinispan.commands.remote.ClusteredGetCommand;
@@ -52,6 +57,8 @@
 import org.infinispan.container.entries.TransientCacheValue;
 import org.infinispan.container.entries.TransientMortalCacheEntry;
 import org.infinispan.container.entries.TransientMortalCacheValue;
+import org.infinispan.distribution.DefaultConsistentHash;
+import org.infinispan.distribution.UnionConsistentHash;
 import org.infinispan.loaders.bucket.Bucket;
 import org.infinispan.marshall.Externalizer;
 import org.infinispan.marshall.Marshallable;
@@ -142,6 +149,11 @@
       MARSHALLABLES.add(RollbackCommand.class.getName());
       MARSHALLABLES.add(InvalidateL1Command.class.getName());
       MARSHALLABLES.add(LockControlCommand.class.getName());
+      MARSHALLABLES.add(GetConsistentHashCommand.class.getName());
+      MARSHALLABLES.add(PushStateCommand.class.getName());
+      MARSHALLABLES.add(PullStateCommand.class.getName());
+      MARSHALLABLES.add(InstallConsistentHashCommand.class.getName());
+      MARSHALLABLES.add(JoinCompleteCommand.class.getName());
 
       MARSHALLABLES.add(ImmortalCacheEntry.class.getName());
       MARSHALLABLES.add(MortalCacheEntry.class.getName());
@@ -160,6 +172,8 @@
       MARSHALLABLES.add(PutOperation.class.getName());
       MARSHALLABLES.add(RemoveOperation.class.getName());
       MARSHALLABLES.add(ClearOperation.class.getName());
+      MARSHALLABLES.add(DefaultConsistentHash.class.getName());
+      MARSHALLABLES.add(UnionConsistentHash.class.getName());
    }
 
    /**
@@ -198,9 +212,9 @@
                Marshallable marshallable = ReflectionUtil.getAnnotation(clazz, Marshallable.class);
                if (marshallable != null && !marshallable.externalizer().equals(Externalizer.class)) {
                   int id = marshallable.id();
-                  Externalizer ext = (Externalizer) Util.getInstance(marshallable.externalizer());
+                  Externalizer ext = Util.getInstance(marshallable.externalizer());
                   if (!ids.add(id))
-                     throw new CacheException("Duplicat id found! id=" + id + " in " + ext.getClass().getName() + " is shared by another marshallable class.");
+                     throw new CacheException("Duplicate id found! id=" + id + " in " + ext.getClass().getName() + " is shared by another marshallable class.");
                   if (ext instanceof ReplicableCommandExternalizer) {
                      ((ReplicableCommandExternalizer) ext).inject(cmdFactory);
                   }

Modified: trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifier.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifier.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifier.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -21,7 +21,7 @@
     * Notifies all registered listeners of a viewChange event.  Note that viewChange notifications are ALWAYS sent
     * immediately.
     */
-   void notifyViewChange(List<Address> members, Address myAddress, int viewId);
+   void notifyViewChange(List<Address> members, List<Address> oldMembers, Address myAddress, int viewId);
 
    void notifyCacheStarted(String cacheName);
 

Modified: trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierImpl.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierImpl.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierImpl.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -55,12 +55,13 @@
       this.cacheManager = cacheManager;
    }
 
-   public void notifyViewChange(List<Address> members, Address myAddress, int viewId) {
+   public void notifyViewChange(List<Address> members, List<Address> oldMembers, Address myAddress, int viewId) {
       if (!viewChangedListeners.isEmpty()) {
          EventImpl e = new EventImpl();
          e.setLocalAddress(myAddress);
          e.setViewId(viewId);
-         e.setNewMemberList(members);
+         e.setNewMembers(members);
+         e.setOldMembers(oldMembers);
          e.setCacheManager(cacheManager);
          e.setType(Event.Type.VIEW_CHANGED);
          for (ListenerInvocation listener : viewChangedListeners) listener.invoke(e);

Modified: trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/event/EventImpl.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/event/EventImpl.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/event/EventImpl.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -16,18 +16,19 @@
    String cacheName;
    CacheManager cacheManager;
    Type type;
-   List<Address> newMemberList;
+   List<Address> newMembers, oldMembers;
    Address localAddress;
    int viewId;
 
    public EventImpl() {
    }
 
-   public EventImpl(String cacheName, CacheManager cacheManager, Type type, List<Address> newMemberList, Address localAddress, int viewId) {
+   public EventImpl(String cacheName, CacheManager cacheManager, Type type, List<Address> newMemberList, List<Address> oldMemberList, Address localAddress, int viewId) {
       this.cacheName = cacheName;
       this.cacheManager = cacheManager;
       this.type = type;
-      this.newMemberList = newMemberList;
+      this.newMembers = newMemberList;
+      this.oldMembers = oldMemberList;
       this.localAddress = localAddress;
       this.viewId = viewId;
    }
@@ -56,14 +57,22 @@
       this.type = type;
    }
 
-   public List<Address> getNewMemberList() {
-      return newMemberList;
+   public List<Address> getNewMembers() {
+      return newMembers;
    }
 
-   public void setNewMemberList(List<Address> newMemberList) {
-      this.newMemberList = newMemberList;
+   public void setNewMembers(List<Address> newMembers) {
+      this.newMembers = newMembers;
    }
 
+   public void setOldMembers(List<Address> oldMembers) {
+      this.oldMembers = oldMembers;
+   }
+
+   public List<Address> getOldMembers() {
+      return this.oldMembers;
+   }
+
    public Address getLocalAddress() {
       return localAddress;
    }
@@ -90,8 +99,10 @@
       if (viewId != event.viewId) return false;
       if (cacheName != null ? !cacheName.equals(event.cacheName) : event.cacheName != null) return false;
       if (localAddress != null ? !localAddress.equals(event.localAddress) : event.localAddress != null) return false;
-      if (newMemberList != null ? !newMemberList.equals(event.newMemberList) : event.newMemberList != null)
+      if (newMembers != null ? !newMembers.equals(event.newMembers) : event.newMembers != null)
          return false;
+      if (oldMembers != null ? !oldMembers.equals(event.oldMembers) : event.oldMembers != null)
+         return false;
       if (type != event.type) return false;
 
       return true;
@@ -101,7 +112,8 @@
    public int hashCode() {
       int result = cacheName != null ? cacheName.hashCode() : 0;
       result = 31 * result + (type != null ? type.hashCode() : 0);
-      result = 31 * result + (newMemberList != null ? newMemberList.hashCode() : 0);
+      result = 31 * result + (newMembers != null ? newMembers.hashCode() : 0);
+      result = 31 * result + (oldMembers != null ? oldMembers.hashCode() : 0);
       result = 31 * result + (localAddress != null ? localAddress.hashCode() : 0);
       result = 31 * result + viewId;
       return result;
@@ -112,7 +124,8 @@
       return "EventImpl{" +
             "cacheName='" + cacheName + '\'' +
             ", type=" + type +
-            ", newMemberList=" + newMemberList +
+            ", newMembers=" + newMembers +
+            ", oldMembers=" + oldMembers +
             ", localAddress=" + localAddress +
             ", viewId=" + viewId +
             '}';

Modified: trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/event/ViewChangedEvent.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/event/ViewChangedEvent.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/notifications/cachemanagerlistener/event/ViewChangedEvent.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -35,8 +35,10 @@
    /**
     * @return the new view associated with this view change.
     */
-   List<Address> getNewMemberList();
+   List<Address> getNewMembers();
 
+   List<Address> getOldMembers();
+
    Address getLocalAddress();
 
    int getViewId();

Modified: trunk/core/src/main/java/org/infinispan/remoting/InboundInvocationHandlerImpl.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/remoting/InboundInvocationHandlerImpl.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/remoting/InboundInvocationHandlerImpl.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -63,6 +63,7 @@
       commandsFactory.initializeReplicableCommand(cmd);
 
       try {
+         log.trace("Calling perform() on {0}", cmd);
          Object retval = cmd.perform(null);
          return cr.getComponent(ResponseGenerator.class).getResponse(cmd, retval);
       } catch (Exception e) {

Modified: trunk/core/src/main/java/org/infinispan/remoting/responses/DistributionResponseGenerator.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/remoting/responses/DistributionResponseGenerator.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/remoting/responses/DistributionResponseGenerator.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -5,6 +5,8 @@
 import org.infinispan.commands.remote.ClusteredGetCommand;
 import org.infinispan.commands.remote.SingleRpcCommand;
 import org.infinispan.commands.write.WriteCommand;
+import org.infinispan.distribution.DistributionManager;
+import org.infinispan.factories.annotations.Inject;
 
 /**
  * A response generator for the DIST cache mode
@@ -13,8 +15,18 @@
  * @since 4.0
  */
 public class DistributionResponseGenerator implements ResponseGenerator {
+   DistributionManager distributionManager;
+
+   @Inject
+   public void inject(DistributionManager distributionManager) {
+      this.distributionManager = distributionManager;
+   }
+
    public Response getResponse(CacheRpcCommand command, Object returnValue) {
       if (command instanceof ClusteredGetCommand) {
+         ClusteredGetCommand clusteredGet = (ClusteredGetCommand) command;
+         if (distributionManager.isAffectedByRehash(clusteredGet.getKey()))
+            return new UnsureResponse();
          return returnValue == null ? null : new SuccessfulResponse(returnValue);
       } else if (command instanceof SingleRpcCommand) {
          SingleRpcCommand src = (SingleRpcCommand) command;

Added: trunk/core/src/main/java/org/infinispan/remoting/responses/UnsureResponse.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/remoting/responses/UnsureResponse.java	                        (rev 0)
+++ trunk/core/src/main/java/org/infinispan/remoting/responses/UnsureResponse.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,14 @@
+package org.infinispan.remoting.responses;
+
+/**
+ * An unsure response - used with Dist - essentially asks the caller to check the next response from the next node since
+ * the sender is in a state of flux (probably in the middle of rebalancing)
+ *
+ * @author Manik Surtani
+ * @since 4.0
+ */
+public class UnsureResponse extends ValidResponse {
+   public boolean isSuccessful() {
+      return false;
+   }
+}


Property changes on: trunk/core/src/main/java/org/infinispan/remoting/responses/UnsureResponse.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Modified: trunk/core/src/main/java/org/infinispan/remoting/rpc/RpcManager.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/remoting/rpc/RpcManager.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/remoting/rpc/RpcManager.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -29,6 +29,7 @@
 import org.infinispan.statetransfer.StateTransferException;
 import org.infinispan.util.concurrent.NotifyingNotifiableFuture;
 
+import java.util.Collection;
 import java.util.List;
 
 /**
@@ -53,7 +54,7 @@
     * @return a list of responses from each member contacted.
     * @throws Exception in the event of problems.
     */
-   List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue, ResponseFilter responseFilter);
+   List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue, ResponseFilter responseFilter);
 
    /**
     * Invokes an RPC call on other caches in the cluster.
@@ -68,7 +69,7 @@
     * @return a list of responses from each member contacted.
     * @throws Exception in the event of problems.
     */
-   List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue);
+   List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue);
 
    /**
     * Invokes an RPC call on other caches in the cluster.
@@ -81,7 +82,7 @@
     * @return a list of responses from each member contacted.
     * @throws Exception in the event of problems.
     */
-   List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout) throws Exception;
+   List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout) throws Exception;
 
    /**
     * Initiates a state retrieval process from neighbouring caches.  This method will block until it either times out,
@@ -143,7 +144,7 @@
     * @param sync       if true, the transport will operate in sync mode.  Otherwise, it will operate in async mode.
     * @throws ReplicationException in the event of problems
     */
-   void invokeRemotely(List<Address> recipients, ReplicableCommand rpc, boolean sync) throws ReplicationException;
+   void invokeRemotely(Collection<Address> recipients, ReplicableCommand rpc, boolean sync) throws ReplicationException;
 
    /**
     * Broadcasts an RPC command to a specified set of recipients
@@ -155,34 +156,34 @@
     * @param usePriorityQueue if true, a priority queue is used
     * @throws ReplicationException in the event of problems
     */
-   void invokeRemotely(List<Address> recipients, ReplicableCommand rpc, boolean sync, boolean usePriorityQueue) throws ReplicationException;
+   void invokeRemotely(Collection<Address> recipients, ReplicableCommand rpc, boolean sync, boolean usePriorityQueue) throws ReplicationException;
 
    /**
-    * The same as {@link #invokeRemotely(java.util.List, org.infinispan.commands.ReplicableCommand, boolean)} except
-    * that the task is passed to the transport executor and a Future is returned.  The transport always deals with this
-    * synchronously.
+    * The same as {@link #invokeRemotely(java.util.Collection, org.infinispan.commands.ReplicableCommand, boolean)}
+    * except that the task is passed to the transport executor and a Future is returned.  The transport always deals
+    * with this synchronously.
     *
     * @param recipients recipients to invoke remote call on
     * @param rpc        command to execute remotely
     * @param future     the future which will be passed back to the user
     */
-   void invokeRemotelyInFuture(List<Address> recipients, ReplicableCommand rpc, NotifyingNotifiableFuture<Object> future);
+   void invokeRemotelyInFuture(Collection<Address> recipients, ReplicableCommand rpc, NotifyingNotifiableFuture<Object> future);
 
    /**
-    * The same as {@link #invokeRemotely(java.util.List, org.infinispan.commands.ReplicableCommand, boolean)} except
-    * that the task is passed to the transport executor and a Future is returned.  The transport always deals with this
-    * synchronously.
+    * The same as {@link #invokeRemotely(java.util.Collection, org.infinispan.commands.ReplicableCommand, boolean)}
+    * except that the task is passed to the transport executor and a Future is returned.  The transport always deals
+    * with this synchronously.
     *
     * @param recipients       recipients to invoke remote call on
     * @param rpc              command to execute remotely
     * @param usePriorityQueue if true, a priority queue is used
     * @param future           the future which will be passed back to the user
     */
-   void invokeRemotelyInFuture(List<Address> recipients, ReplicableCommand rpc, boolean usePriorityQueue, NotifyingNotifiableFuture<Object> future);
+   void invokeRemotelyInFuture(Collection<Address> recipients, ReplicableCommand rpc, boolean usePriorityQueue, NotifyingNotifiableFuture<Object> future);
 
    /**
-    * The same as {@link #invokeRemotelyInFuture(java.util.List, org.infinispan.commands.ReplicableCommand, boolean,
-    * org.infinispan.util.concurrent.NotifyingNotifiableFuture)} except that you can specify a timeout.
+    * The same as {@link #invokeRemotelyInFuture(java.util.Collection, org.infinispan.commands.ReplicableCommand,
+    * boolean, org.infinispan.util.concurrent.NotifyingNotifiableFuture)} except that you can specify a timeout.
     *
     * @param recipients       recipients to invoke remote call on
     * @param rpc              command to execute remotely
@@ -190,7 +191,7 @@
     * @param future           the future which will be passed back to the user
     * @param timeout          after which to give up (in millis)
     */
-   void invokeRemotelyInFuture(final List<Address> recipients, final ReplicableCommand rpc, final boolean usePriorityQueue, final NotifyingNotifiableFuture<Object> future, final long timeout);
+   void invokeRemotelyInFuture(final Collection<Address> recipients, final ReplicableCommand rpc, final boolean usePriorityQueue, final NotifyingNotifiableFuture<Object> future, final long timeout);
 
    /**
     * @return a reference to the underlying transport.

Modified: trunk/core/src/main/java/org/infinispan/remoting/rpc/RpcManagerImpl.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/remoting/rpc/RpcManagerImpl.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/remoting/rpc/RpcManagerImpl.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -23,6 +23,7 @@
 import org.infinispan.util.logging.LogFactory;
 
 import java.text.NumberFormat;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
@@ -76,7 +77,7 @@
       return !sync && replicationQueue != null && replicationQueue.isEnabled();
    }
 
-   public final List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue, ResponseFilter responseFilter) {
+   public final List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue, ResponseFilter responseFilter) {
       List<Address> members = t.getMembers();
       if (members.size() < 2) {
          if (log.isDebugEnabled())
@@ -102,11 +103,11 @@
       }
    }
 
-   public final List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue) {
+   public final List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue) {
       return invokeRemotely(recipients, rpcCommand, mode, timeout, usePriorityQueue, null);
    }
 
-   public final List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout) throws Exception {
+   public final List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout) throws Exception {
       return invokeRemotely(recipients, rpcCommand, mode, timeout, false, null);
    }
 
@@ -192,15 +193,15 @@
       invokeRemotelyInFuture(null, rpc, usePriorityQueue, l);
    }
 
-   public final void invokeRemotely(List<Address> recipients, ReplicableCommand rpc, boolean sync) throws ReplicationException {
+   public final void invokeRemotely(Collection<Address> recipients, ReplicableCommand rpc, boolean sync) throws ReplicationException {
       invokeRemotely(recipients, rpc, sync, false);
    }
 
-   public final void invokeRemotely(List<Address> recipients, ReplicableCommand rpc, boolean sync, boolean usePriorityQueue) throws ReplicationException {
+   public final void invokeRemotely(Collection<Address> recipients, ReplicableCommand rpc, boolean sync, boolean usePriorityQueue) throws ReplicationException {
       invokeRemotely(recipients, rpc, sync, usePriorityQueue, configuration.getSyncReplTimeout());
    }
 
-   public final void invokeRemotely(List<Address> recipients, ReplicableCommand rpc, boolean sync, boolean usePriorityQueue, long timeout) throws ReplicationException {
+   public final void invokeRemotely(Collection<Address> recipients, ReplicableCommand rpc, boolean sync, boolean usePriorityQueue, long timeout) throws ReplicationException {
       if (trace) log.trace("{0} broadcasting call {1} to recipient list {2}", t.getAddress(), rpc, recipients);
 
       if (useReplicationQueue(sync)) {
@@ -216,15 +217,15 @@
       }
    }
 
-   public final void invokeRemotelyInFuture(List<Address> recipients, ReplicableCommand rpc, NotifyingNotifiableFuture<Object> l) {
+   public final void invokeRemotelyInFuture(Collection<Address> recipients, ReplicableCommand rpc, NotifyingNotifiableFuture<Object> l) {
       invokeRemotelyInFuture(recipients, rpc, false, l);
    }
 
-   public final void invokeRemotelyInFuture(final List<Address> recipients, final ReplicableCommand rpc, final boolean usePriorityQueue, final NotifyingNotifiableFuture<Object> l) {
+   public final void invokeRemotelyInFuture(final Collection<Address> recipients, final ReplicableCommand rpc, final boolean usePriorityQueue, final NotifyingNotifiableFuture<Object> l) {
       invokeRemotelyInFuture(recipients, rpc, usePriorityQueue, l, configuration.getSyncReplTimeout());
    }
 
-   public final void invokeRemotelyInFuture(final List<Address> recipients, final ReplicableCommand rpc, final boolean usePriorityQueue, final NotifyingNotifiableFuture<Object> l, final long timeout) {
+   public final void invokeRemotelyInFuture(final Collection<Address> recipients, final ReplicableCommand rpc, final boolean usePriorityQueue, final NotifyingNotifiableFuture<Object> l, final long timeout) {
       Callable<Object> c = new Callable<Object>() {
          public Object call() {
             invokeRemotely(recipients, rpc, true, usePriorityQueue, timeout);

Modified: trunk/core/src/main/java/org/infinispan/remoting/transport/Transport.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/remoting/transport/Transport.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/remoting/transport/Transport.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -19,6 +19,7 @@
 import org.infinispan.remoting.rpc.ResponseMode;
 import org.infinispan.statetransfer.StateTransferException;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 
@@ -63,7 +64,7 @@
     * @return a list of responses from each member contacted.
     * @throws Exception in the event of problems.
     */
-   List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout,
+   List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout,
                                  boolean usePriorityQueue, ResponseFilter responseFilter, boolean supportReplay) throws Exception;
 
    /**

Modified: trunk/core/src/main/java/org/infinispan/remoting/transport/jgroups/JGroupsTransport.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/remoting/transport/jgroups/JGroupsTransport.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/remoting/transport/jgroups/JGroupsTransport.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -38,6 +38,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Properties;
@@ -144,8 +145,6 @@
       // Channel.LOCAL *must* be set to false so we don't see our own messages - otherwise invalidations targeted at
       // remote instances will be received by self.
       channel.setOpt(Channel.LOCAL, false);
-      channel.setOpt(Channel.AUTO_RECONNECT, true);
-      channel.setOpt(Channel.AUTO_GETSTATE, false);
       channel.setOpt(Channel.BLOCK, true);
       dispatcher = new CommandAwareRpcDispatcher(channel, this,
                                                  asyncExecutor, inboundInvocationHandler, flushTracker, distributedSyncTimeout);
@@ -283,7 +282,7 @@
    // outbound RPC
    // ------------------------------------------------------------------------------------------------------------------
 
-   public List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout,
+   public List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout,
                                         boolean usePriorityQueue, ResponseFilter responseFilter, boolean supportReplay)
          throws Exception {
 
@@ -309,8 +308,8 @@
 
          if (mode.isAsynchronous()) return Collections.emptyList();// async case
 
-         if (trace)
-            log.trace("Cache [{0}], is caller thread interupted? {3}: responses for command {1}:\n{2}", getAddress(), rpcCommand.getClass().getSimpleName(), rsps, Thread.currentThread().isInterrupted());
+//         if (trace)
+//            log.trace("Cache [{0}], is caller thread interupted? {3}: responses for command {1}:\n{2}", getAddress(), rpcCommand.getClass().getSimpleName(), rsps, Thread.currentThread().isInterrupted());
 
          // short-circuit no-return-value calls.
          if (rsps == null) return Collections.emptyList();
@@ -373,6 +372,7 @@
 
    public void viewAccepted(View newView) {
       Vector<org.jgroups.Address> newMembers = newView.getMembers();
+      List<Address> oldMembers = null;
       if (log.isInfoEnabled()) log.info("Received new cluster view: {0}", newView);
       synchronized (membersListLock) {
          boolean needNotification = false;
@@ -392,7 +392,8 @@
 //                  }
 //            }
 
-            // we need a defensive copy anyway
+            oldMembers = members;
+            // we need a defensive copy anyway            
             members = fromJGroupsAddressList(newMembers);
             needNotification = true;
          }
@@ -401,7 +402,7 @@
 
          // now notify listeners - *after* updating the coordinator. - JBCACHE-662
          if (needNotification && notifier != null) {
-            notifier.notifyViewChange(members, getAddress(), (int) newView.getVid().getId());
+            notifier.notifyViewChange(members, oldMembers, getAddress(), (int) newView.getVid().getId());
          }
 
          // Wake up any threads that are waiting to know about who the coordinator is
@@ -485,7 +486,7 @@
    // Helpers to convert between Address types
    // ------------------------------------------------------------------------------------------------------------------
 
-   private Vector<org.jgroups.Address> toJGroupsAddressVector(List<Address> list) {
+   private Vector<org.jgroups.Address> toJGroupsAddressVector(Collection<Address> list) {
       if (list == null) return null;
       if (list.isEmpty()) return new Vector<org.jgroups.Address>();
 

Modified: trunk/core/src/main/java/org/infinispan/util/Util.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/util/Util.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/util/Util.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -176,7 +176,7 @@
 
       }
    }
-   
+
    public static void flushAndCloseStream(OutputStream o) {
       if (o == null) return;
       try {
@@ -191,7 +191,7 @@
 
       }
    }
-   
+
    public static void flushAndCloseOutput(ObjectOutput o) {
       if (o == null) return;
       try {
@@ -206,5 +206,4 @@
 
       }
    }
-
 }

Modified: trunk/core/src/main/java/org/infinispan/util/concurrent/ReclosableLatch.java
===================================================================
--- trunk/core/src/main/java/org/infinispan/util/concurrent/ReclosableLatch.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/main/java/org/infinispan/util/concurrent/ReclosableLatch.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -34,7 +34,7 @@
  * @since 4.0
  */
 public class ReclosableLatch extends AbstractQueuedSynchronizer {
-   
+
    private static final long serialVersionUID = 1744280161777661090l;
 
    // the following states are used in the AQS.
@@ -78,4 +78,11 @@
    public final boolean await(long time, TimeUnit unit) throws InterruptedException {
       return tryAcquireSharedNanos(1, unit.toNanos(time)); // the 1 is a dummy value that is not used.
    }
+
+   @Override
+   public String toString() {
+      int s = getState();
+      String q = hasQueuedThreads() ? "non" : "";
+      return "ReclosableLatch [State = " + s + ", " + q + "empty queue]";
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/AsyncAPIAsyncDistTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/AsyncAPIAsyncDistTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/AsyncAPIAsyncDistTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,47 +1,50 @@
 package org.infinispan.distribution;
 
-// For some reason I had to comment this out to ensure the test was disabled!
+import org.infinispan.commands.write.WriteCommand;
+import org.infinispan.test.AbstractCacheTest;
+import org.infinispan.test.ReplListener;
+import org.testng.annotations.Test;
 
-//@Test(groups = "functional", testName = "distribution.AsyncAPIAsyncDistTest", enabled = false)
+ at Test(groups = "functional", testName = "distribution.AsyncAPIAsyncDistTest")
 public class AsyncAPIAsyncDistTest extends AsyncAPISyncDistTest {
 
-//   ReplListener rl;
-//
-//   public AsyncAPIAsyncDistTest() {
-//      cleanup = AbstractCacheTest.CleanupPhase.AFTER_METHOD;
-//   }
-//
-//   @Override
-//   protected void createCacheManagers() throws Throwable {
-//      super.createCacheManagers();
-//      rl = new ReplListener(c2, true);
-//   }
-//
-//   @Override
-//   protected boolean sync() {
-//      return false;
-//   }
-//
-//   @Override
-//   protected void resetListeners() {
-//      rl.resetEager();
-//   }
-//
-//   @Override
-//   protected void asyncWait(boolean tx, Class<? extends WriteCommand>... cmds) {
-//      if (tx) {
-//         if (cmds == null || cmds.length == 0)
-//            rl.expectAnyWithTx();
-//         else
-//            rl.expectWithTx(cmds);
-//      } else {
-//         if (cmds == null || cmds.length == 0)
-//            rl.expectAny();
-//         else
-//            rl.expect(cmds);
-//      }
-//
-//
-//      rl.waitForRpc();
-//   }
+   ReplListener rl;
+
+   public AsyncAPIAsyncDistTest() {
+      cleanup = AbstractCacheTest.CleanupPhase.AFTER_METHOD;
+   }
+
+   @Override
+   protected void createCacheManagers() throws Throwable {
+      super.createCacheManagers();
+      rl = new ReplListener(c2, true);
+   }
+
+   @Override
+   protected boolean sync() {
+      return false;
+   }
+
+   @Override
+   protected void resetListeners() {
+      rl.resetEager();
+   }
+
+   @Override
+   protected void asyncWait(boolean tx, Class<? extends WriteCommand>... cmds) {
+      if (tx) {
+         if (cmds == null || cmds.length == 0)
+            rl.expectAnyWithTx();
+         else
+            rl.expectWithTx(cmds);
+      } else {
+         if (cmds == null || cmds.length == 0)
+            rl.expectAny();
+         else
+            rl.expect(cmds);
+      }
+
+
+      rl.waitForRpc();
+   }
 }
\ No newline at end of file

Modified: trunk/core/src/test/java/org/infinispan/distribution/AsyncAPISyncDistTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/AsyncAPISyncDistTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/AsyncAPISyncDistTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,27 +1,34 @@
 package org.infinispan.distribution;
 
+import org.infinispan.Cache;
+import org.infinispan.config.Configuration;
+import org.infinispan.context.Flag;
 import org.infinispan.replication.AsyncAPISyncReplTest;
+import org.infinispan.test.data.Key;
+import org.infinispan.transaction.lookup.DummyTransactionManagerLookup;
+import org.infinispan.util.Util;
+import org.testng.annotations.Test;
 
-// For some reason I had to comment this out to ensure the test was disabled!
+import java.util.List;
 
-//@Test(groups = "functional", testName = "distribution.AsyncAPISyncDistTest", enabled = false)
+ at Test(groups = "functional", testName = "distribution.AsyncAPISyncDistTest")
 public class AsyncAPISyncDistTest extends AsyncAPISyncReplTest {
 
-//   @SuppressWarnings("unchecked")
-//   @Override
-//   protected void createCacheManagers() throws Throwable {
-//      Configuration c =
-//            getDefaultClusteredConfig(sync() ? Configuration.CacheMode.DIST_SYNC : Configuration.CacheMode.DIST_ASYNC);
-//      c.setTransactionManagerLookupClass(DummyTransactionManagerLookup.class.getName());
-//      List<Cache<Key, String>> l = createClusteredCaches(2, getClass().getSimpleName(), c);
-//      c1 = l.get(0);
-//      c2 = l.get(1);
-//   }
-//
-//   @Override
-//   protected void assertOnAllCaches(Key k, String v) {
-//      Object real;
-//      assert Util.safeEquals((real = c1.getAdvancedCache().get(k, Flag.SKIP_REMOTE_LOOKUP)), v) : "Error on cache 1.  Expected " + v + " and got " + real;
-//      assert Util.safeEquals((real = c2.getAdvancedCache().get(k, Flag.SKIP_REMOTE_LOOKUP)), v) : "Error on cache 2.  Expected " + v + " and got " + real;
-//   }
+   @SuppressWarnings("unchecked")
+   @Override
+   protected void createCacheManagers() throws Throwable {
+      Configuration c =
+            getDefaultClusteredConfig(sync() ? Configuration.CacheMode.DIST_SYNC : Configuration.CacheMode.DIST_ASYNC);
+      c.setTransactionManagerLookupClass(DummyTransactionManagerLookup.class.getName());
+      List<Cache<Key, String>> l = createClusteredCaches(2, getClass().getSimpleName(), c);
+      c1 = l.get(0);
+      c2 = l.get(1);
+   }
+
+   @Override
+   protected void assertOnAllCaches(Key k, String v) {
+      Object real;
+      assert Util.safeEquals((real = c1.getAdvancedCache().get(k, Flag.SKIP_REMOTE_LOOKUP)), v) : "Error on cache 1.  Expected " + v + " and got " + real;
+      assert Util.safeEquals((real = c2.getAdvancedCache().get(k, Flag.SKIP_REMOTE_LOOKUP)), v) : "Error on cache 2.  Expected " + v + " and got " + real;
+   }
 }
\ No newline at end of file

Modified: trunk/core/src/test/java/org/infinispan/distribution/BaseDistCacheStoreTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/BaseDistCacheStoreTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/BaseDistCacheStoreTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -21,6 +21,12 @@
  */
 package org.infinispan.distribution;
 
+import org.infinispan.config.CacheLoaderManagerConfig;
+import org.infinispan.config.Configuration;
+import org.infinispan.loaders.dummy.DummyInMemoryCacheStore;
+import org.infinispan.manager.CacheManager;
+import org.infinispan.test.fwk.TestCacheManagerFactory;
+
 /**
  * DistSyncCacheStoreTest.
  *
@@ -28,18 +34,18 @@
  * @since 4.0
  */
 public abstract class BaseDistCacheStoreTest extends BaseDistFunctionalTest {
-//   boolean shared;
-//   int id;
-//
-//   @Override
-//   protected CacheManager addClusterEnabledCacheManager() {
-//      Configuration cfg = new Configuration();
-//      CacheLoaderManagerConfig clmc = new CacheLoaderManagerConfig();
-//      clmc.setShared(shared);
-//      clmc.addCacheLoaderConfig(new DummyInMemoryCacheStore.Cfg(getClass().getSimpleName() + "_" + id++));
-//      cfg.setCacheLoaderManagerConfig(clmc);
-//      CacheManager cm = TestCacheManagerFactory.createClusteredCacheManager(cfg);
-//      cacheManagers.add(cm);
-//      return cm;
-//   }
+   boolean shared;
+   int id;
+
+   @Override
+   protected CacheManager addClusterEnabledCacheManager() {
+      Configuration cfg = new Configuration();
+      CacheLoaderManagerConfig clmc = new CacheLoaderManagerConfig();
+      clmc.setShared(shared);
+      clmc.addCacheLoaderConfig(new DummyInMemoryCacheStore.Cfg(getClass().getSimpleName() + "_" + id++));
+      cfg.setCacheLoaderManagerConfig(clmc);
+      CacheManager cm = TestCacheManagerFactory.createClusteredCacheManager(cfg);
+      cacheManagers.add(cm);
+      return cm;
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/BaseDistFunctionalTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/BaseDistFunctionalTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/BaseDistFunctionalTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,241 +1,349 @@
 package org.infinispan.distribution;
 
+import org.infinispan.Cache;
+import org.infinispan.commands.VisitableCommand;
+import org.infinispan.commands.write.PutKeyValueCommand;
+import org.infinispan.config.Configuration;
+import org.infinispan.container.DataContainer;
+import org.infinispan.container.entries.ImmortalCacheEntry;
+import org.infinispan.container.entries.InternalCacheEntry;
+import org.infinispan.container.entries.MortalCacheEntry;
+import org.infinispan.manager.CacheManager;
+import org.infinispan.remoting.transport.Address;
 import org.infinispan.test.MultipleCacheManagersTest;
+import org.infinispan.test.TestingUtil;
+import org.infinispan.transaction.lookup.DummyTransactionManagerLookup;
+import org.infinispan.util.Util;
+import org.infinispan.util.concurrent.IsolationLevel;
+import org.testng.annotations.Test;
 
-//@Test(groups = "functional", testName = "distribution.BaseDistFunctionalTest", enabled = false)
+import javax.transaction.TransactionManager;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+ at Test(groups = "functional", testName = "distribution.BaseDistFunctionalTest")
 public abstract class BaseDistFunctionalTest extends MultipleCacheManagersTest {
-   protected void createCacheManagers() {
+   protected String cacheName;
+   protected Cache<Object, String> c1, c2, c3, c4;
+   protected Configuration configuration;
+   protected List<Cache<Object, String>> caches;
+   protected List<Address> cacheAddresses;
+   protected boolean sync = true;
+   protected boolean tx = false;
+   protected boolean testRetVals = true;
 
+   protected void createCacheManagers() throws Throwable {
+      cacheName = "dist";
+      configuration = getDefaultClusteredConfig(sync ? Configuration.CacheMode.DIST_SYNC : Configuration.CacheMode.DIST_ASYNC);
+      if (!testRetVals) {
+         configuration.setUnsafeUnreliableReturnValues(true);
+         // we also need to use repeatable read for tests to work when we dont have reliable return values, since the
+         // tests repeatedly queries changes
+         configuration.setIsolationLevel(IsolationLevel.REPEATABLE_READ);
+      }
+      if (tx) configuration.setTransactionManagerLookupClass(DummyTransactionManagerLookup.class.getName());
+      caches = createClusteredCaches(4, cacheName, configuration);
+
+      reorderBasedOnCHPositions();
+
+      c1 = caches.get(0);
+      c2 = caches.get(1);
+      c3 = caches.get(2);
+      c4 = caches.get(3);
+
+      cacheAddresses = new ArrayList<Address>(4);
+      for (Cache cache : caches) cacheAddresses.add(cache.getCacheManager().getAddress());
    }
 
-//   protected Cache<Object, String> c1, c2, c3, c4;
-//   protected List<Cache<Object, String>> caches;
-//   protected boolean sync = true;
-//   protected boolean tx = false;
-//   protected boolean testRetVals = true;
-//
-//   protected void createCacheManagers() throws Throwable {
-//      Configuration c = getDefaultClusteredConfig(sync ? DIST_SYNC : DIST_ASYNC);
-//      if (!testRetVals) {
-//         c.setUnsafeUnreliableReturnValues(true);
-//         // we also need to use repeatable read for tests to work when we dont have reliable return values, since the
-//         // tests repeatedly queries changes
-//         c.setIsolationLevel(IsolationLevel.REPEATABLE_READ);
-//      }
-//      if (tx) c.setTransactionManagerLookupClass(DummyTransactionManagerLookup.class.getName());
-//      caches = createClusteredCaches(4, "dist", c);
-//      c1 = caches.get(0);
-//      c2 = caches.get(1);
-//      c3 = caches.get(2);
-//      c4 = caches.get(3);
-//   }
-//
-//   // ----------------- HELPERS ----------------
-//
-//   protected void initAndTest() {
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//
-//      c1.put("k1", "value");
-//      asyncWait("k1", PutKeyValueCommand.class, getNonOwnersExcludingSelf("k1", addressOf(c1)));
-//      for (Cache<Object, String> c : caches)
-//         assert "value".equals(c.get("k1")) : "Failed on cache " + addressOf(c);
-//      assertOwnershipAndNonOwnership("k1");
-//   }
-//
-//   protected static Address addressOf(Cache<?, ?> cache) {
-//      return cache.getCacheManager().getAddress();
-//   }
-//
-//   protected Cache<Object, String> getFirstNonOwner(String key) {
-//      return getNonOwners(key)[0];
-//   }
-//
-//   protected Cache<Object, String> getSecondNonOwner(String key) {
-//      return getNonOwners(key)[1];
-//   }
-//
-//   protected void assertOnAllCachesAndOwnership(Object key, String value) {
-//      for (Cache<Object, String> c : caches) {
-//         Object realVal = c.get(key);
-//         if (value == null) {
-//            assert realVal == null : "Expecting [" + key + "] to equal [" + value + "] on cache ["
-//                  + addressOf(c) + "] but was [" + realVal + "]";
-//         } else {
-//            assert value.equals(realVal) : "Expecting [" + key + "] to equal [" + value + "] on cache ["
-//                  + addressOf(c) + "] but was [" + realVal + "]";
-//         }
-//      }
-//      if (value != null) assertOwnershipAndNonOwnership(key);
-//   }
-//
-//   protected void assertOwnershipAndNonOwnership(Object key) {
-//      for (Cache<Object, String> c : caches) {
-//         DataContainer dc = c.getAdvancedCache().getDataContainer();
-//         if (isOwner(c, key)) {
-//            InternalCacheEntry ice = dc.get(key);
-//            assert ice != null : "Fail on cache " + addressOf(c) + ": dc.get(" + key + ") returned null!";
-//            assert ice instanceof ImmortalCacheEntry : "Fail on cache " + addressOf(c) + ": dc.get(" + key + ") returned " + dc.get(key);
-//         } else {
-//            if (dc.containsKey(key)) {
-//               assert dc.get(key) instanceof MortalCacheEntry : "Fail on cache " + addressOf(c) + ": dc.get(" + key + ") returned " + dc.get(key);
-//               assert dc.get(key).getLifespan() == c1.getConfiguration().getL1Lifespan();
-//            }
-//         }
-//      }
-//   }
-//
-//   protected void assertIsInL1(Cache<?, ?> cache, Object key) {
-//      DataContainer dc = cache.getAdvancedCache().getDataContainer();
-//      InternalCacheEntry ice = dc.get(key);
-//      assert ice != null : "Entry for key [" + key + "] should be in data container on cache at [" + addressOf(cache) + "]!";
-//      assert !(ice instanceof ImmortalCacheEntry) : "Entry for key [" + key + "] should have a lifespan on cache at [" + addressOf(cache) + "]!";
-//   }
-//
-//   protected void assertIsNotInL1(Cache<?, ?> cache, Object key) {
-//      DataContainer dc = cache.getAdvancedCache().getDataContainer();
-//      InternalCacheEntry ice = dc.get(key);
-//      assert ice == null : "Entry for key [" + key + "] should not be in data container on cache at [" + addressOf(cache) + "]!";
-//   }
-//
-//   protected void assertIsInContainerImmortal(Cache<?, ?> cache, Object key) {
-//      DataContainer dc = cache.getAdvancedCache().getDataContainer();
-//      InternalCacheEntry ice = dc.get(key);
-//      assert ice != null : "Entry for key [" + key + "] should be in data container on cache at [" + addressOf(cache) + "]!";
-//      assert ice instanceof ImmortalCacheEntry : "Entry for key [" + key + "] on cache at [" + addressOf(cache) + "] should be immortal but was [" + ice + "]!";
-//   }
-//
-//   protected static boolean isOwner(Cache<?, ?> c, Object key) {
-//      DistributionManager dm = c.getAdvancedCache().getComponentRegistry().getComponent(DistributionManager.class);
-//      List<Address> ownerAddresses = dm.locate(key);
-//      for (Address a : ownerAddresses) {
-//         if (addressOf(c).equals(a)) return true;
-//      }
-//      return false;
-//   }
-//
-//   protected static boolean isFirstOwner(Cache<?, ?> c, Object key) {
-//      DistributionManager dm = c.getAdvancedCache().getComponentRegistry().getComponent(DistributionManager.class);
-//      List<Address> ownerAddresses = dm.locate(key);
-//      return addressOf(c).equals(ownerAddresses.get(0));
-//   }
-//
-//   protected Cache<Object, String>[] getOwners(Object key) {
-//      Cache<Object, String>[] owners = new Cache[2];
-//      int i = 0;
-//      for (Cache<Object, String> c : caches) {
-//         if (isOwner(c, key)) owners[i++] = c;
-//      }
-//      return owners;
-//   }
-//
-//   protected Cache<Object, String>[] getNonOwnersExcludingSelf(Object key, Address self) {
-//      Cache<Object, String>[] nonOwners = getNonOwners(key);
-//      boolean selfInArray = false;
-//      for (Cache<?, ?> c : nonOwners) {
-//         if (addressOf(c).equals(self)) {
-//            selfInArray = true;
-//            break;
-//         }
-//      }
-//
-//      if (selfInArray) {
-//         Cache<Object, String>[] nonOwnersExclSelf = new Cache[nonOwners.length - 1];
-//         int i = 0;
-//         for (Cache<Object, String> c : nonOwners) {
-//            if (!addressOf(c).equals(self)) nonOwnersExclSelf[i++] = c;
-//         }
-//         return nonOwnersExclSelf;
-//      } else {
-//         return nonOwners;
-//      }
-//   }
-//
-//   protected Cache<Object, String>[] getNonOwners(Object key) {
-//      Cache<Object, String>[] nonOwners = new Cache[2];
-//      int i = 0;
-//      for (Cache<Object, String> c : caches) {
-//         if (!isOwner(c, key)) nonOwners[i++] = c;
-//      }
-//      return nonOwners;
-//   }
-//
-//   protected List<Address> residentAddresses(Object key) {
-//      DistributionManager dm = c1.getAdvancedCache().getComponentRegistry().getComponent(DistributionManager.class);
-//      return dm.locate(key);
-//   }
-//
-//   protected DistributionManager getDistributionManager(Cache<?, ?> c) {
-//      return c.getAdvancedCache().getComponentRegistry().getComponent(DistributionManager.class);
-//   }
-//
-//   /**
-//    * Blocks and waits for a replication event on async caches
-//    *
-//    * @param key     key that causes the replication.  Used to determine which caches to listen on.  If null, all caches
-//    *                are checked
-//    * @param command command to listen for
-//    * @param caches  on which this key should be invalidated
-//    */
-//   protected void asyncWait(Object key, Class<? extends VisitableCommand> command, Cache<?, ?>... caches) {
-//      // no op.
-//   }
-//
-//   protected TransactionManager getTransactionManager(Cache<?, ?> cache) {
-//      return TestingUtil.getTransactionManager(cache);
-//   }
-//
-//   /**
-//    * A special type of key that if passed a cache in its constructor, will ensure it will always be assigned to that
-//    * cache (plus however many additional caches in the hash space)
-//    */
-//   protected static class MagicKey implements Serializable {
-//      int hashcode;
-//      String address;
-//
-//      public MagicKey(Cache<?, ?> toMapTo) {
-//         address = addressOf(toMapTo).toString();
-//         for (int i = 0; i < toMapTo.getCacheManager().getMembers().size(); i++) {
-//            // create a dummy object with this hashcode
-//            final int hc = i;
-//            Object dummy = new Object() {
-//               @Override
-//               public int hashCode() {
-//                  return hc;
-//               }
-//            };
-//
-//            if (BaseDistFunctionalTest.isFirstOwner(toMapTo, dummy)) {
-//               // we have found a hashcode that works!
-//               hashcode = hc;
-//               break;
-//            }
-//         }
-//      }
-//
-//      @Override
-//      public int hashCode() {
-//         return hashcode;
-//      }
-//
-//      @Override
-//      public boolean equals(Object o) {
-//         if (this == o) return true;
-//         if (o == null || getClass() != o.getClass()) return false;
-//
-//         MagicKey magicKey = (MagicKey) o;
-//
-//         if (hashcode != magicKey.hashcode) return false;
-//         if (address != null ? !address.equals(magicKey.address) : magicKey.address != null) return false;
-//
-//         return true;
-//      }
-//
-//      @Override
-//      public String toString() {
-//         return "MagicKey{" +
-//               "hashcode=" + hashcode +
-//               ", address='" + address + '\'' +
-//               '}';
-//      }
-//   }
+
+   // only used if the CH impl does not order the hash ring based on the order of the view.
+   // in the case of the DefaultConsistentHash, the order is based on a has code of the addres modded by
+   // the hash space.  So this will not adhere to the positions in the view, but it is deterministic.
+   // so this function orders things such that the test can predict where keys get mapped to.
+   private void reorderBasedOnCHPositions() {
+      // wait for all joiners to join
+      List<Cache> clist = new ArrayList<Cache>(cacheManagers.size());
+      for (CacheManager cm : cacheManagers) clist.add(cm.getCache(cacheName));
+      assert clist.size() == 4;
+      waitForJoinTasksToComplete(120000, clist.toArray(new Cache[clist.size()]));
+
+      // seed this with an initial cache.  Any one will do.
+      Cache seed = caches.get(0);
+      DefaultConsistentHash ch = getDefaultConsistentHash(seed, 120000);
+      List<Cache<Object, String>> reordered = new ArrayList<Cache<Object, String>>();
+
+      for (Address a : ch.positions.values()) {
+         for (Cache<Object, String> c : caches) {
+            if (a.equals(c.getCacheManager().getAddress())) {
+               reordered.add(c);
+               break;
+            }
+         }
+      }
+
+      caches = reordered;
+   }
+
+   // ----------------- HELPERS ----------------
+
+   protected void waitForJoinTasksToComplete(long timeout, Cache... joiners) {
+      long giveupTime = System.currentTimeMillis() + timeout;
+      while (System.currentTimeMillis() < giveupTime) {
+         boolean allOK = true;
+         for (Cache c : joiners) {
+            DistributionManagerImpl dmi = (DistributionManagerImpl) getDistributionManager(c);
+            allOK &= dmi.joinComplete;
+         }
+         if (allOK) return;
+         TestingUtil.sleepThread(100);
+      }
+      throw new RuntimeException("Some caches have not finished rehashing after " + Util.prettyPrintTime(timeout));
+   }
+
+
+   protected void initAndTest() {
+      System.out.println("Caches are " + cacheAddresses);
+
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+
+      c1.put("k1", "value");
+      asyncWait("k1", PutKeyValueCommand.class, getNonOwnersExcludingSelf("k1", addressOf(c1)));
+      for (Cache<Object, String> c : caches)
+         assert "value".equals(c.get("k1")) : "Failed on cache " + addressOf(c);
+      assertOwnershipAndNonOwnership("k1");
+   }
+
+   protected static Address addressOf(Cache<?, ?> cache) {
+      return cache.getCacheManager().getAddress();
+   }
+
+   protected Cache<Object, String> getFirstNonOwner(String key) {
+      return getNonOwners(key)[0];
+   }
+
+   protected Cache<Object, String> getSecondNonOwner(String key) {
+      return getNonOwners(key)[1];
+   }
+
+   protected void assertOnAllCachesAndOwnership(Object key, String value) {
+      for (Cache<Object, String> c : caches) {
+         Object realVal = c.get(key);
+         if (value == null) {
+            assert realVal == null : "Expecting [" + key + "] to equal [" + value + "] on cache ["
+                  + addressOf(c) + "] but was [" + realVal + "]";
+         } else {
+            assert value.equals(realVal) : "Expecting [" + key + "] to equal [" + value + "] on cache ["
+                  + addressOf(c) + "] but was [" + realVal + "]";
+         }
+      }
+      if (value != null) assertOwnershipAndNonOwnership(key);
+   }
+
+   protected void assertOwnershipAndNonOwnership(Object key) {
+      for (Cache<Object, String> c : caches) {
+         DataContainer dc = c.getAdvancedCache().getDataContainer();
+         if (isOwner(c, key)) {
+            InternalCacheEntry ice = dc.get(key);
+            assert ice != null : "Fail on cache " + addressOf(c) + ": dc.get(" + key + ") returned null!";
+            assert ice instanceof ImmortalCacheEntry : "Fail on cache " + addressOf(c) + ": dc.get(" + key + ") returned " + safeType(dc.get(key));
+         } else {
+            if (dc.containsKey(key)) {
+               assert dc.get(key) instanceof MortalCacheEntry : "Fail on cache " + addressOf(c) + ": dc.get(" + key + ") returned " + safeType(dc.get(key));
+               assert dc.get(key).getLifespan() == c1.getConfiguration().getL1Lifespan();
+            }
+         }
+      }
+   }
+
+   protected static final String safeType(Object o) {
+      if (o == null) return "null";
+      return o.getClass().getSimpleName();
+   }
+
+   protected void assertIsInL1(Cache<?, ?> cache, Object key) {
+      DataContainer dc = cache.getAdvancedCache().getDataContainer();
+      InternalCacheEntry ice = dc.get(key);
+      assert ice != null : "Entry for key [" + key + "] should be in data container on cache at [" + addressOf(cache) + "]!";
+      assert !(ice instanceof ImmortalCacheEntry) : "Entry for key [" + key + "] should have a lifespan on cache at [" + addressOf(cache) + "]!";
+   }
+
+   protected void assertIsNotInL1(Cache<?, ?> cache, Object key) {
+      DataContainer dc = cache.getAdvancedCache().getDataContainer();
+      InternalCacheEntry ice = dc.get(key);
+      assert ice == null : "Entry for key [" + key + "] should not be in data container on cache at [" + addressOf(cache) + "]!";
+   }
+
+   protected void assertIsInContainerImmortal(Cache<?, ?> cache, Object key) {
+      DataContainer dc = cache.getAdvancedCache().getDataContainer();
+      InternalCacheEntry ice = dc.get(key);
+      assert ice != null : "Entry for key [" + key + "] should be in data container on cache at [" + addressOf(cache) + "]!";
+      assert ice instanceof ImmortalCacheEntry : "Entry for key [" + key + "] on cache at [" + addressOf(cache) + "] should be immortal but was [" + ice + "]!";
+   }
+
+   protected static boolean isOwner(Cache<?, ?> c, Object key) {
+      DistributionManager dm = c.getAdvancedCache().getComponentRegistry().getComponent(DistributionManager.class);
+      List<Address> ownerAddresses = dm.locate(key);
+      for (Address a : ownerAddresses) {
+         if (addressOf(c).equals(a)) return true;
+      }
+      return false;
+   }
+
+   protected static boolean isFirstOwner(Cache<?, ?> c, Object key) {
+      DistributionManager dm = c.getAdvancedCache().getComponentRegistry().getComponent(DistributionManager.class);
+      List<Address> ownerAddresses = dm.locate(key);
+      return addressOf(c).equals(ownerAddresses.get(0));
+   }
+
+   protected Cache<Object, String>[] getOwners(Object key) {
+      Cache<Object, String>[] owners = new Cache[2];
+      int i = 0;
+      for (Cache<Object, String> c : caches) {
+         if (isOwner(c, key)) owners[i++] = c;
+      }
+      return owners;
+   }
+
+   protected Cache<Object, String>[] getNonOwnersExcludingSelf(Object key, Address self) {
+      Cache<Object, String>[] nonOwners = getNonOwners(key);
+      boolean selfInArray = false;
+      for (Cache<?, ?> c : nonOwners) {
+         if (addressOf(c).equals(self)) {
+            selfInArray = true;
+            break;
+         }
+      }
+
+      if (selfInArray) {
+         Cache<Object, String>[] nonOwnersExclSelf = new Cache[nonOwners.length - 1];
+         int i = 0;
+         for (Cache<Object, String> c : nonOwners) {
+            if (!addressOf(c).equals(self)) nonOwnersExclSelf[i++] = c;
+         }
+         return nonOwnersExclSelf;
+      } else {
+         return nonOwners;
+      }
+   }
+
+   protected Cache<Object, String>[] getNonOwners(Object key) {
+      Cache<Object, String>[] nonOwners = new Cache[2];
+      int i = 0;
+      for (Cache<Object, String> c : caches) {
+         if (!isOwner(c, key)) nonOwners[i++] = c;
+      }
+      return nonOwners;
+   }
+
+   protected List<Address> residentAddresses(Object key) {
+      DistributionManager dm = c1.getAdvancedCache().getComponentRegistry().getComponent(DistributionManager.class);
+      return dm.locate(key);
+   }
+
+   protected DistributionManager getDistributionManager(Cache<?, ?> c) {
+      return c.getAdvancedCache().getComponentRegistry().getComponent(DistributionManager.class);
+   }
+
+   protected DefaultConsistentHash getDefaultConsistentHash(Cache<?, ?> c) {
+      return (DefaultConsistentHash) getDistributionManager(c).getConsistentHash();
+   }
+
+   protected DefaultConsistentHash getDefaultConsistentHash(Cache<?, ?> c, long timeout) {
+      long expTime = System.currentTimeMillis() + timeout;
+      while (System.currentTimeMillis() < expTime) {
+         ConsistentHash ch = getDistributionManager(c).getConsistentHash();
+         if (ch instanceof DefaultConsistentHash) return (DefaultConsistentHash) ch;
+         TestingUtil.sleepThread(100);
+      }
+      throw new RuntimeException("Timed out waiting for a DefaultConsistentHash to be present on cache [" + addressOf(c) + "]");
+   }
+
+   /**
+    * Blocks and waits for a replication event on async caches
+    *
+    * @param key     key that causes the replication.  Used to determine which caches to listen on.  If null, all caches
+    *                are checked
+    * @param command command to listen for
+    * @param caches  on which this key should be invalidated
+    */
+   protected void asyncWait(Object key, Class<? extends VisitableCommand> command, Cache<?, ?>... caches) {
+      // no op.
+   }
+
+   protected void assertProperConsistentHashOnAllCaches() {
+      // check that ALL caches in the system DON'T have a temporary UnionCH
+      for (Cache c : caches) {
+         DistributionManager dm = getDistributionManager(c);
+         assert !(dm.getConsistentHash() instanceof UnionConsistentHash);
+      }
+   }
+
+   protected TransactionManager getTransactionManager(Cache<?, ?> cache) {
+      return TestingUtil.getTransactionManager(cache);
+   }
+
+   /**
+    * A special type of key that if passed a cache in its constructor, will ensure it will always be assigned to that
+    * cache (plus however many additional caches in the hash space)
+    */
+   protected static class MagicKey implements Serializable {
+      String name = null;
+      int hashcode;
+      String address;
+
+      public MagicKey(Cache<?, ?> toMapTo) {
+         address = addressOf(toMapTo).toString();
+         Random r = new Random();
+         for (; ;) {
+            // create a dummy object with this hashcode
+            final int hc = r.nextInt(DefaultConsistentHash.HASH_SPACE);
+            Object dummy = new Object() {
+               @Override
+               public int hashCode() {
+                  return hc;
+               }
+            };
+
+            if (BaseDistFunctionalTest.isFirstOwner(toMapTo, dummy)) {
+               // we have found a hashcode that works!
+               hashcode = hc;
+               break;
+            }
+         }
+      }
+
+      public MagicKey(Cache<?, ?> toMapTo, String name) {
+         this(toMapTo);
+         this.name = name;
+      }
+
+      @Override
+      public int hashCode() {
+         return hashcode;
+      }
+
+      @Override
+      public boolean equals(Object o) {
+         if (this == o) return true;
+         if (o == null || getClass() != o.getClass()) return false;
+
+         MagicKey magicKey = (MagicKey) o;
+
+         if (hashcode != magicKey.hashcode) return false;
+         if (address != null ? !address.equals(magicKey.address) : magicKey.address != null) return false;
+
+         return true;
+      }
+
+      @Override
+      public String toString() {
+         return "MagicKey{" +
+               (name == null ? "" : "name=" + name) +
+               ", hashcode=" + hashcode +
+               ", address='" + address + '\'' +
+               '}';
+      }
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/DefaultConsistentHashTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DefaultConsistentHashTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DefaultConsistentHashTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -11,7 +11,7 @@
 import java.util.List;
 import java.util.Map;
 
- at Test(groups = "unit", testName = "distribution.DefaultConsistentHashTest", enabled = false)
+ at Test(groups = "unit", testName = "distribution.DefaultConsistentHashTest")
 public class DefaultConsistentHashTest {
 
    List<Address> servers;
@@ -109,6 +109,6 @@
    }
 
    public int compareTo(Object o) {
-      return this.addressNum - ((TestAddress)o).addressNum;
+      return this.addressNum - ((TestAddress) o).addressNum;
    }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistAsyncFuncTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistAsyncFuncTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistAsyncFuncTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,49 +1,58 @@
 package org.infinispan.distribution;
 
-//@Test(groups = "functional", testName = "distribution.DistAsyncFuncTest", enabled = false)
+import org.infinispan.Cache;
+import org.infinispan.commands.VisitableCommand;
+import org.infinispan.commands.write.InvalidateL1Command;
+import org.infinispan.test.ReplListener;
+import org.testng.annotations.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+ at Test(groups = "functional", testName = "distribution.DistAsyncFuncTest")
 public class DistAsyncFuncTest extends DistSyncFuncTest {
 
-//   ReplListener r1, r2, r3, r4;
-//   ReplListener[] r;
-//   Map<Cache<?, ?>, ReplListener> listenerLookup;
-//
-//   public DistAsyncFuncTest() {
-//      sync = false;
-//      tx = false;
-//      testRetVals = true;
-//   }
-//
-//   @Override
-//   protected void createCacheManagers() throws Throwable {
-//      super.createCacheManagers();
-//      r1 = new ReplListener(c1, true, true);
-//      r2 = new ReplListener(c2, true, true);
-//      r3 = new ReplListener(c3, true, true);
-//      r4 = new ReplListener(c4, true, true);
-//      r = new ReplListener[]{r1, r2, r3, r4};
-//      listenerLookup = new HashMap<Cache<?, ?>, ReplListener>();
-//      for (ReplListener rl : r) listenerLookup.put(rl.getCache(), rl);
-//   }
-//
-//
-//   @Override
-//   protected void asyncWait(Object key, Class<? extends VisitableCommand> command, Cache<?, ?>... cachesOnWhichKeyShouldInval) {
-//      if (key == null) {
-//         // test all caches.
-//         for (ReplListener rl : r) rl.expect(command);
-//         for (ReplListener rl : r) rl.waitForRpc();
-//      } else {
-//         for (Cache<?, ?> c : getOwners(key)) {
-//            listenerLookup.get(c).expect(command);
-//            listenerLookup.get(c).waitForRpc();
-//         }
-//
-//         if (cachesOnWhichKeyShouldInval != null) {
-//            for (Cache<?, ?> c : cachesOnWhichKeyShouldInval) {
-//               listenerLookup.get(c).expect(InvalidateL1Command.class);
-//               listenerLookup.get(c).waitForRpc();
-//            }
-//         }
-//      }
-//   }
+   ReplListener r1, r2, r3, r4;
+   ReplListener[] r;
+   Map<Cache<?, ?>, ReplListener> listenerLookup;
+
+   public DistAsyncFuncTest() {
+      sync = false;
+      tx = false;
+      testRetVals = true;
+   }
+
+   @Override
+   protected void createCacheManagers() throws Throwable {
+      super.createCacheManagers();
+      r1 = new ReplListener(c1, true, true);
+      r2 = new ReplListener(c2, true, true);
+      r3 = new ReplListener(c3, true, true);
+      r4 = new ReplListener(c4, true, true);
+      r = new ReplListener[]{r1, r2, r3, r4};
+      listenerLookup = new HashMap<Cache<?, ?>, ReplListener>();
+      for (ReplListener rl : r) listenerLookup.put(rl.getCache(), rl);
+   }
+
+
+   @Override
+   protected void asyncWait(Object key, Class<? extends VisitableCommand> command, Cache<?, ?>... cachesOnWhichKeyShouldInval) {
+      if (key == null) {
+         // test all caches.
+         for (ReplListener rl : r) rl.expect(command);
+         for (ReplListener rl : r) rl.waitForRpc();
+      } else {
+         for (Cache<?, ?> c : getOwners(key)) {
+            listenerLookup.get(c).expect(command);
+            listenerLookup.get(c).waitForRpc();
+         }
+
+         if (cachesOnWhichKeyShouldInval != null) {
+            for (Cache<?, ?> c : cachesOnWhichKeyShouldInval) {
+               listenerLookup.get(c).expect(InvalidateL1Command.class);
+               listenerLookup.get(c).waitForRpc();
+            }
+         }
+      }
+   }
 }
\ No newline at end of file

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistAsyncTxFuncTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistAsyncTxFuncTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistAsyncTxFuncTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,69 +1,83 @@
 package org.infinispan.distribution;
 
-//@Test(groups = "functional", testName = "distribution.DistAsyncTxFuncTest", enabled = false)
+import org.infinispan.Cache;
+import org.infinispan.commands.VisitableCommand;
+import org.infinispan.commands.write.InvalidateL1Command;
+import org.infinispan.test.ReplListener;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+ at Test(groups = "functional", testName = "distribution.DistAsyncTxFuncTest")
 public class DistAsyncTxFuncTest extends DistSyncTxFuncTest {
 
-//   ReplListener r1, r2, r3, r4;
-//   ReplListener[] r;
-//   Map<Cache<?, ?>, ReplListener> listenerLookup;
-//
-//   public DistAsyncTxFuncTest() {
-//      sync = false;
-//      tx = true;
-//      testRetVals = true;
-//      cleanup = CleanupPhase.AFTER_METHOD; // ensure any stale TXs are wiped
-//   }
-//
-//   @Override
-//   protected void createCacheManagers() throws Throwable {
-//      super.createCacheManagers();
-//      r1 = new ReplListener(c1, true, true);
-//      r2 = new ReplListener(c2, true, true);
-//      r3 = new ReplListener(c3, true, true);
-//      r4 = new ReplListener(c4, true, true);
-//      r = new ReplListener[]{r1, r2, r3, r4};
-//      listenerLookup = new HashMap<Cache<?, ?>, ReplListener>();
-//      for (ReplListener rl : r) listenerLookup.put(rl.getCache(), rl);
-//   }
-//
-//   @Override
-//   protected void asyncWait(Object key, Class<? extends VisitableCommand> command, Cache<?, ?>... cachesOnWhichKeyShouldInval) {
-//      if (cachesOnWhichKeyShouldInval == null) cachesOnWhichKeyShouldInval = new Cache[0];
-//      List<Cache<?, ?>> cachesOnWhichKeyShouldInvalList = new ArrayList(Arrays.asList(cachesOnWhichKeyShouldInval));
-//      if (key == null) {
-//         // test all caches.
-//         for (ReplListener rl : r) rl.expect(command);
-//         for (ReplListener rl : r) rl.waitForRpc();
-//      } else {
-//         for (Cache<?, ?> c : getOwners(key)) {
-//            if (cachesOnWhichKeyShouldInvalList.remove(c)) {
-//               listenerLookup.get(c).expect(command, InvalidateL1Command.class);
-//            } else {
-//               listenerLookup.get(c).expect(command);
-//            }
-//            listenerLookup.get(c).waitForRpc();
-//         }
-//
-//         for (Cache<?, ?> c : cachesOnWhichKeyShouldInvalList) {
-//            listenerLookup.get(c).expect(InvalidateL1Command.class);
-//            listenerLookup.get(c).waitForRpc();
-//         }
-//      }
-//   }
-//
-//   @Override
-//   protected void asyncTxWait(Object... keys) {
-//      // Wait for a tx completion event
-//      if (keys != null) {
-//         Set<Cache<?, ?>> cachesInTx = new HashSet<Cache<?, ?>>();
-//         for (Object k : keys) {
-//            cachesInTx.addAll(Arrays.asList(getOwners(k)));
-//         }
-//
-//         for (Cache<?, ?> c : cachesInTx) {
-//            listenerLookup.get(c).expectAnyWithTx();
-//            listenerLookup.get(c).waitForRpc();
-//         }
-//      }
-//   }
+   ReplListener r1, r2, r3, r4;
+   ReplListener[] r;
+   Map<Cache<?, ?>, ReplListener> listenerLookup;
+
+   public DistAsyncTxFuncTest() {
+      sync = false;
+      tx = true;
+      testRetVals = true;
+      cleanup = CleanupPhase.AFTER_METHOD; // ensure any stale TXs are wiped
+   }
+
+   @Override
+   protected void createCacheManagers() throws Throwable {
+      super.createCacheManagers();
+      r1 = new ReplListener(c1, true, true);
+      r2 = new ReplListener(c2, true, true);
+      r3 = new ReplListener(c3, true, true);
+      r4 = new ReplListener(c4, true, true);
+      r = new ReplListener[]{r1, r2, r3, r4};
+      listenerLookup = new HashMap<Cache<?, ?>, ReplListener>();
+      for (ReplListener rl : r) listenerLookup.put(rl.getCache(), rl);
+   }
+
+   @Override
+   protected void asyncWait(Object key, Class<? extends VisitableCommand> command, Cache<?, ?>... cachesOnWhichKeyShouldInval) {
+      if (cachesOnWhichKeyShouldInval == null) cachesOnWhichKeyShouldInval = new Cache[0];
+      List<Cache<?, ?>> cachesOnWhichKeyShouldInvalList = new ArrayList(Arrays.asList(cachesOnWhichKeyShouldInval));
+      if (key == null) {
+         // test all caches.
+         for (ReplListener rl : r) rl.expect(command);
+         for (ReplListener rl : r) rl.waitForRpc();
+      } else {
+         for (Cache<?, ?> c : getOwners(key)) {
+            if (cachesOnWhichKeyShouldInvalList.remove(c)) {
+               listenerLookup.get(c).expect(command, InvalidateL1Command.class);
+            } else {
+               listenerLookup.get(c).expect(command);
+            }
+            listenerLookup.get(c).waitForRpc();
+         }
+
+         for (Cache<?, ?> c : cachesOnWhichKeyShouldInvalList) {
+            listenerLookup.get(c).expect(InvalidateL1Command.class);
+            listenerLookup.get(c).waitForRpc();
+         }
+      }
+   }
+
+   @Override
+   protected void asyncTxWait(Object... keys) {
+      // Wait for a tx completion event
+      if (keys != null) {
+         Set<Cache<?, ?>> cachesInTx = new HashSet<Cache<?, ?>>();
+         for (Object k : keys) {
+            cachesInTx.addAll(Arrays.asList(getOwners(k)));
+         }
+
+         for (Cache<?, ?> c : cachesInTx) {
+            listenerLookup.get(c).expectAnyWithTx();
+            listenerLookup.get(c).waitForRpc();
+         }
+      }
+   }
 }
\ No newline at end of file

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistAsyncTxUnsafeFuncTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistAsyncTxUnsafeFuncTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistAsyncTxUnsafeFuncTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,11 +1,13 @@
 package org.infinispan.distribution;
 
-//@Test(groups = "functional", testName = "distribution.DistAsyncTxUnsafeFuncTest", enabled = false)
+import org.testng.annotations.Test;
+
+ at Test(groups = "functional", testName = "distribution.DistAsyncTxUnsafeFuncTest")
 public class DistAsyncTxUnsafeFuncTest extends DistAsyncTxFuncTest {
-//   public DistAsyncTxUnsafeFuncTest() {
-//      sync = false;
-//      tx = true;
-//      testRetVals = false;
-//      cleanup = CleanupPhase.AFTER_METHOD; // ensure any stale TXs are wiped
-//   }
+   public DistAsyncTxUnsafeFuncTest() {
+      sync = false;
+      tx = true;
+      testRetVals = false;
+      cleanup = CleanupPhase.AFTER_METHOD; // ensure any stale TXs are wiped
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistAsyncUnsafeFuncTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistAsyncUnsafeFuncTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistAsyncUnsafeFuncTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,10 +1,12 @@
 package org.infinispan.distribution;
 
-//@Test(groups = "functional", testName = "distribution.DistAsyncUnsafeFuncTest", enabled = false)
+import org.testng.annotations.Test;
+
+ at Test(groups = "functional", testName = "distribution.DistAsyncUnsafeFuncTest")
 public class DistAsyncUnsafeFuncTest extends DistAsyncFuncTest {
-//   public DistAsyncUnsafeFuncTest() {
-//      sync = false;
-//      tx = false;
-//      testRetVals = false;
-//   }
+   public DistAsyncUnsafeFuncTest() {
+      sync = false;
+      tx = false;
+      testRetVals = false;
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistSkipRemoteLookupTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistSkipRemoteLookupTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistSkipRemoteLookupTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,49 +1,52 @@
 package org.infinispan.distribution;
 
-//@Test(groups = "functional", testName = "distribution.DistSkipRemoteLookupTest", enabled = false)
+import static org.infinispan.context.Flag.SKIP_REMOTE_LOOKUP;
+import org.testng.annotations.Test;
+
+ at Test(groups = "functional", testName = "distribution.DistSkipRemoteLookupTest")
 public class DistSkipRemoteLookupTest extends BaseDistFunctionalTest {
-//   public void testSkipLookupOnGet() {
-//      MagicKey k1 = new MagicKey(c1);
-//      c1.put(k1, "value");
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsNotInL1(c3, k1);
-//      assertIsNotInL1(c4, k1);
-//
-//      assert c4.getAdvancedCache().get(k1, SKIP_REMOTE_LOOKUP) == null;
-//
-//      assertOwnershipAndNonOwnership(k1);
-//   }
-//
-//   public void testCorrectFunctionalityOnConditionalWrite() {
-//      MagicKey k1 = new MagicKey(c1);
-//      c1.put(k1, "value");
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsNotInL1(c3, k1);
-//      assertIsNotInL1(c4, k1);
-//
-//      assert c4.getAdvancedCache().putIfAbsent(k1, "new_val", SKIP_REMOTE_LOOKUP) == null;
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsNotInL1(c3, k1);
-//      assertIsInL1(c4, k1);
-//   }
-//
-//   public void testCorrectFunctionalityOnUnconditionalWrite() {
-//      MagicKey k1 = new MagicKey(c1);
-//      c1.put(k1, "value");
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsNotInL1(c3, k1);
-//      assertIsNotInL1(c4, k1);
-//
-//      assert c4.getAdvancedCache().put(k1, "new_val", SKIP_REMOTE_LOOKUP) == null;
-//      assert c3.get(k1).equals("new_val");
-//      assertOnAllCachesAndOwnership(k1, "new_val");
-//   }
+   public void testSkipLookupOnGet() {
+      MagicKey k1 = new MagicKey(c1);
+      c1.put(k1, "value");
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsNotInL1(c3, k1);
+      assertIsNotInL1(c4, k1);
+
+      assert c4.getAdvancedCache().get(k1, SKIP_REMOTE_LOOKUP) == null;
+
+      assertOwnershipAndNonOwnership(k1);
+   }
+
+   public void testCorrectFunctionalityOnConditionalWrite() {
+      MagicKey k1 = new MagicKey(c1);
+      c1.put(k1, "value");
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsNotInL1(c3, k1);
+      assertIsNotInL1(c4, k1);
+
+      assert c4.getAdvancedCache().putIfAbsent(k1, "new_val", SKIP_REMOTE_LOOKUP) == null;
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsNotInL1(c3, k1);
+      assertIsInL1(c4, k1);
+   }
+
+   public void testCorrectFunctionalityOnUnconditionalWrite() {
+      MagicKey k1 = new MagicKey(c1);
+      c1.put(k1, "value");
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsNotInL1(c3, k1);
+      assertIsNotInL1(c4, k1);
+
+      assert c4.getAdvancedCache().put(k1, "new_val", SKIP_REMOTE_LOOKUP) == null;
+      assert c3.get(k1).equals("new_val");
+      assertOnAllCachesAndOwnership(k1, "new_val");
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistSyncCacheStoreNotSharedTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistSyncCacheStoreNotSharedTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistSyncCacheStoreNotSharedTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -21,146 +21,161 @@
  */
 package org.infinispan.distribution;
 
+import org.infinispan.Cache;
+import org.infinispan.commands.write.ClearCommand;
+import org.infinispan.commands.write.PutKeyValueCommand;
+import org.infinispan.commands.write.RemoveCommand;
+import org.infinispan.commands.write.ReplaceCommand;
+import org.infinispan.loaders.CacheLoaderManager;
+import org.infinispan.loaders.CacheStore;
+import org.infinispan.test.TestingUtil;
+import org.infinispan.util.logging.Log;
+import org.infinispan.util.logging.LogFactory;
+import org.testng.annotations.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * DistSyncSharedTest.
  *
  * @author Galder Zamarreño
  * @since 4.0
  */
-//@Test(groups = "functional", testName = "distribution.DistSyncCacheStoreNotSharedTest", enabled = true)
+ at Test(groups = "functional", testName = "distribution.DistSyncCacheStoreNotSharedTest", enabled = true)
 public class DistSyncCacheStoreNotSharedTest extends BaseDistCacheStoreTest {
-//   private static final Log log = LogFactory.getLog(DistSyncCacheStoreNotSharedTest.class);
-//
-//   public DistSyncCacheStoreNotSharedTest() {
-//      sync = true;
-//      tx = false;
-//      testRetVals = true;
-//      shared = false;
-//   }
-//
-//   public void testPutFromNonOwner() throws Exception {
-//      String key = "k2", value = "value2";
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//      Cache<Object, String> nonOwner = getFirstNonOwner(key);
-//      CacheStore nonOwnerStore = TestingUtil.extractComponent(nonOwner, CacheLoaderManager.class).getCacheStore();
-//      assert !nonOwnerStore.containsKey(key);
-//      Object retval = nonOwner.put(key, value);
-//      asyncWait(key, PutKeyValueCommand.class, getSecondNonOwner(key));
-//      assert !nonOwnerStore.containsKey(key);
-//      if (testRetVals) assert retval == null;
-//      assertOnAllCachesAndOwnership(key, value);
-//   }
-//
-//   public void testPutFromOwner() throws Exception {
-//      String key = "k3", value = "value3";
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//      getOwners(key)[0].put(key, value);
-//      asyncWait(key, PutKeyValueCommand.class, getNonOwners(key));
-//      for (Cache<Object, String> c : caches) {
-//         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         if (isOwner(c, key)) {
-//            assertIsInContainerImmortal(c, key);
-//            assert store.containsKey(key);
-//         } else {
-//            assertIsNotInL1(c, key);
-//            assert !store.containsKey(key);
-//         }
-//      }
-//   }
-//
-//   public void testPutAll() throws Exception {
-//      String k1 = "1", v1 = "one", k2 = "2", v2 = "two", k3 = "3", v3 = "three", k4 = "4", v4 = "four";
-//      String[] keys = new String[]{k1, k2, k3, k4};
-//      Map<String, String> data = new HashMap<String, String>();
-//      data.put(k1, v1);
-//      data.put(k2, v2);
-//      data.put(k3, v3);
-//      data.put(k4, v4);
-//
-//      c1.putAll(data);
-//
-//      for (String key : keys) {
-//         for (Cache<Object, String> c : caches) {
-//            CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//            if (isOwner(c, key)) {
-//               assertIsInContainerImmortal(c, key);
-//               assert store.containsKey(key);
-//            } else {
-//               assert !store.containsKey(key);
-//            }
-//         }
-//      }
-//   }
-//
-//   public void testRemoveFromNonOwner() throws Exception {
-//      String key = "k1", value = "value";
-//      initAndTest();
-//
-//      for (Cache<Object, String> c : caches) {
-//         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         if (isOwner(c, key)) {
-//            assertIsInContainerImmortal(c, key);
-//            assert store.load(key).getValue().equals(value);
-//         } else {
-//            assert !store.containsKey(key);
-//         }
-//      }
-//
-//      Object retval = getFirstNonOwner(key).remove(key);
-//      asyncWait("k1", RemoveCommand.class, getSecondNonOwner("k1"));
-//      if (testRetVals) assert "value".equals(retval);
-//      for (Cache<Object, String> c : caches) {
-//         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         assert !store.containsKey(key);
-//      }
-//   }
-//
-//   public void testReplaceFromNonOwner() throws Exception {
-//      String key = "k1", value = "value", value2 = "v2";
-//      initAndTest();
-//
-//      for (Cache<Object, String> c : caches) {
-//         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         if (isOwner(c, key)) {
-//            assertIsInContainerImmortal(c, key);
-//            assert store.load(key).getValue().equals(value);
-//         } else {
-//            assert !store.containsKey(key);
-//         }
-//      }
-//
-//      Object retval = getFirstNonOwner(key).replace(key, value2);
-//      asyncWait(key, ReplaceCommand.class, getSecondNonOwner(key));
-//      if (testRetVals) assert value.equals(retval);
-//      for (Cache<Object, String> c : caches) {
-//         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         if (isOwner(c, key)) {
-//            assertIsInContainerImmortal(c, key);
-//            assert store.load(key).getValue().equals(value2);
-//         } else {
-//            assert !store.containsKey(key);
-//         }
-//      }
-//   }
-//
-//   public void testClear() throws Exception {
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//      for (int i = 0; i < 5; i++) {
-//         getOwners("k" + i)[0].put("k" + i, "value" + i);
-//         asyncWait("k" + i, PutKeyValueCommand.class, getNonOwners("k" + i));
-//      }
-//      // this will fill up L1 as well
-//      for (int i = 0; i < 5; i++) assertOnAllCachesAndOwnership("k" + i, "value" + i);
-//      for (Cache<Object, String> c : caches) assert !c.isEmpty();
-//      c1.clear();
-//      asyncWait(null, ClearCommand.class);
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//      for (int i = 0; i < 5; i++) {
-//         String key = "k" + i;
-//         for (Cache<Object, String> c : caches) {
-//            CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//            assert !store.containsKey(key);
-//         }
-//      }
-//   }
+   private static final Log log = LogFactory.getLog(DistSyncCacheStoreNotSharedTest.class);
+
+   public DistSyncCacheStoreNotSharedTest() {
+      sync = true;
+      tx = false;
+      testRetVals = true;
+      shared = false;
+   }
+
+   public void testPutFromNonOwner() throws Exception {
+      String key = "k2", value = "value2";
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+      Cache<Object, String> nonOwner = getFirstNonOwner(key);
+      CacheStore nonOwnerStore = TestingUtil.extractComponent(nonOwner, CacheLoaderManager.class).getCacheStore();
+      assert !nonOwnerStore.containsKey(key);
+      Object retval = nonOwner.put(key, value);
+      asyncWait(key, PutKeyValueCommand.class, getSecondNonOwner(key));
+      assert !nonOwnerStore.containsKey(key);
+      if (testRetVals) assert retval == null;
+      assertOnAllCachesAndOwnership(key, value);
+   }
+
+   public void testPutFromOwner() throws Exception {
+      String key = "k3", value = "value3";
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+      getOwners(key)[0].put(key, value);
+      asyncWait(key, PutKeyValueCommand.class, getNonOwners(key));
+      for (Cache<Object, String> c : caches) {
+         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         if (isOwner(c, key)) {
+            assertIsInContainerImmortal(c, key);
+            assert store.containsKey(key);
+         } else {
+            assertIsNotInL1(c, key);
+            assert !store.containsKey(key);
+         }
+      }
+   }
+
+   public void testPutAll() throws Exception {
+      String k1 = "1", v1 = "one", k2 = "2", v2 = "two", k3 = "3", v3 = "three", k4 = "4", v4 = "four";
+      String[] keys = new String[]{k1, k2, k3, k4};
+      Map<String, String> data = new HashMap<String, String>();
+      data.put(k1, v1);
+      data.put(k2, v2);
+      data.put(k3, v3);
+      data.put(k4, v4);
+
+      c1.putAll(data);
+
+      for (String key : keys) {
+         for (Cache<Object, String> c : caches) {
+            CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+            if (isOwner(c, key)) {
+               assertIsInContainerImmortal(c, key);
+               assert store.containsKey(key);
+            } else {
+               assert !store.containsKey(key);
+            }
+         }
+      }
+   }
+
+   public void testRemoveFromNonOwner() throws Exception {
+      String key = "k1", value = "value";
+      initAndTest();
+
+      for (Cache<Object, String> c : caches) {
+         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         if (isOwner(c, key)) {
+            assertIsInContainerImmortal(c, key);
+            assert store.load(key).getValue().equals(value);
+         } else {
+            assert !store.containsKey(key);
+         }
+      }
+
+      Object retval = getFirstNonOwner(key).remove(key);
+      asyncWait("k1", RemoveCommand.class, getSecondNonOwner("k1"));
+      if (testRetVals) assert "value".equals(retval);
+      for (Cache<Object, String> c : caches) {
+         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         assert !store.containsKey(key);
+      }
+   }
+
+   public void testReplaceFromNonOwner() throws Exception {
+      String key = "k1", value = "value", value2 = "v2";
+      initAndTest();
+
+      for (Cache<Object, String> c : caches) {
+         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         if (isOwner(c, key)) {
+            assertIsInContainerImmortal(c, key);
+            assert store.load(key).getValue().equals(value);
+         } else {
+            assert !store.containsKey(key);
+         }
+      }
+
+      Object retval = getFirstNonOwner(key).replace(key, value2);
+      asyncWait(key, ReplaceCommand.class, getSecondNonOwner(key));
+      if (testRetVals) assert value.equals(retval);
+      for (Cache<Object, String> c : caches) {
+         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         if (isOwner(c, key)) {
+            assertIsInContainerImmortal(c, key);
+            assert store.load(key).getValue().equals(value2);
+         } else {
+            assert !store.containsKey(key);
+         }
+      }
+   }
+
+   public void testClear() throws Exception {
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+      for (int i = 0; i < 5; i++) {
+         getOwners("k" + i)[0].put("k" + i, "value" + i);
+         asyncWait("k" + i, PutKeyValueCommand.class, getNonOwners("k" + i));
+      }
+      // this will fill up L1 as well
+      for (int i = 0; i < 5; i++) assertOnAllCachesAndOwnership("k" + i, "value" + i);
+      for (Cache<Object, String> c : caches) assert !c.isEmpty();
+      c1.clear();
+      asyncWait(null, ClearCommand.class);
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+      for (int i = 0; i < 5; i++) {
+         String key = "k" + i;
+         for (Cache<Object, String> c : caches) {
+            CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+            assert !store.containsKey(key);
+         }
+      }
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistSyncCacheStoreSharedTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistSyncCacheStoreSharedTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistSyncCacheStoreSharedTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -21,177 +21,190 @@
  */
 package org.infinispan.distribution;
 
+import org.infinispan.Cache;
+import org.infinispan.commands.write.ClearCommand;
+import org.infinispan.commands.write.PutKeyValueCommand;
+import org.infinispan.commands.write.RemoveCommand;
+import org.infinispan.commands.write.ReplaceCommand;
+import org.infinispan.loaders.CacheLoaderManager;
+import org.infinispan.loaders.CacheStore;
+import org.infinispan.test.TestingUtil;
+import org.testng.annotations.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * DistSyncCacheStoreSharedTest.
  *
  * @author Galder Zamarreño
  * @since 4.0
  */
-//@Test(groups = "functional", testName = "distribution.DistSyncCacheStoreSharedTest", enabled = true)
+ at Test(groups = "functional", testName = "distribution.DistSyncCacheStoreSharedTest", enabled = true)
 public class DistSyncCacheStoreSharedTest extends BaseDistCacheStoreTest {
 
-//   public DistSyncCacheStoreSharedTest() {
-//      sync = true;
-//      tx = false;
-//      testRetVals = true;
-//      shared = true;
-//   }
-//
-//   public void testPutFromNonOwner() throws Exception {
-//      String key = "k4", value = "value4";
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//      Cache<Object, String> nonOwner = getFirstNonOwner(key);
-//      CacheStore nonOwnerStore = TestingUtil.extractComponent(nonOwner, CacheLoaderManager.class).getCacheStore();
-//      assert !nonOwnerStore.containsKey(key);
-//      Object retval = nonOwner.put(key, value);
-//      asyncWait(key, PutKeyValueCommand.class, getSecondNonOwner(key));
-//
-//      Cache[] owners = getOwners(key);
-//      CacheStore store = TestingUtil.extractComponent(owners[0], CacheLoaderManager.class).getCacheStore();
-//      assertIsInContainerImmortal(owners[0], key);
-//      assert store.containsKey(key);
-//
-//      for (int i = 1; i < owners.length; i++) {
-//         store = TestingUtil.extractComponent(owners[i], CacheLoaderManager.class).getCacheStore();
-//         assertIsInContainerImmortal(owners[i], key);
-//         assert !store.containsKey(key);
-//      }
-//
-//      for (Cache<Object, String> c : caches) {
-//         store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         if (!isOwner(c, key)) {
-//            assert !store.containsKey(key);
-//         }
-//      }
-//
-//      if (testRetVals) assert retval == null;
-//      assertOnAllCachesAndOwnership(key, value);
-//   }
-//
-//   public void testPutFromOwner() throws Exception {
-//      String key = "k5", value = "value5";
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//      Cache[] owners = getOwners(key);
-//      Object retval = owners[0].put(key, value);
-//      asyncWait(key, PutKeyValueCommand.class, getNonOwners(key));
-//      CacheStore store = TestingUtil.extractComponent(owners[0], CacheLoaderManager.class).getCacheStore();
-//      assertIsInContainerImmortal(owners[0], key);
-//      assert store.containsKey(key);
-//
-//      for (int i = 1; i < owners.length; i++) {
-//         store = TestingUtil.extractComponent(owners[i], CacheLoaderManager.class).getCacheStore();
-//         assertIsInContainerImmortal(owners[i], key);
-//         assert !store.containsKey(key);
-//      }
-//
-//      for (Cache<Object, String> c : caches) {
-//         store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         if (!isOwner(c, key)) {
-//            assert !store.containsKey(key);
-//         }
-//      }
-//
-//      if (testRetVals) assert retval == null;
-//      assertOnAllCachesAndOwnership(key, value);
-//   }
-//
-//
-//   public void testPutAll() throws Exception {
-//      String k1 = "1", v1 = "one", k2 = "2", v2 = "two", k3 = "3", v3 = "three", k4 = "4", v4 = "four";
-//      String[] keys = new String[]{k1, k2, k3, k4};
-//      Map<String, String> data = new HashMap<String, String>();
-//      data.put(k1, v1);
-//      data.put(k2, v2);
-//      data.put(k3, v3);
-//      data.put(k4, v4);
-//
-//      c1.putAll(data);
-//
-//      for (String key : keys) {
-//         for (Cache<Object, String> c : caches) {
-//            CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//            if (isFirstOwner(c, key)) {
-//               assertIsInContainerImmortal(c, key);
-//               assert store.containsKey(key);
-//            } else {
-//               assert !store.containsKey(key);
-//            }
-//         }
-//      }
-//   }
-//
-//   public void testRemoveFromNonOwner() throws Exception {
-//      String key = "k1", value = "value";
-//      initAndTest();
-//
-//      for (Cache<Object, String> c : caches) {
-//         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         if (isFirstOwner(c, key)) {
-//            assertIsInContainerImmortal(c, key);
-//            assert store.load(key).getValue().equals(value);
-//         } else {
-//            assert !store.containsKey(key);
-//         }
-//      }
-//
-//      Object retval = getFirstNonOwner(key).remove(key);
-//      asyncWait("k1", RemoveCommand.class, getSecondNonOwner("k1"));
-//      if (testRetVals) assert value.equals(retval);
-//      for (Cache<Object, String> c : caches) {
-//         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         assert !store.containsKey(key);
-//      }
-//   }
-//
-//   public void testReplaceFromNonOwner() throws Exception {
-//      String key = "k1", value = "value", value2 = "v2";
-//      initAndTest();
-//
-//      for (Cache<Object, String> c : caches) {
-//         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         if (isFirstOwner(c, key)) {
-//            assertIsInContainerImmortal(c, key);
-//            assert store.load(key).getValue().equals(value);
-//         } else {
-//            assert !store.containsKey(key);
-//         }
-//      }
-//
-//      Object retval = getFirstNonOwner(key).replace(key, value2);
-//      asyncWait(key, ReplaceCommand.class, getSecondNonOwner(key));
-//      if (testRetVals) assert value.equals(retval);
-//      for (Cache<Object, String> c : caches) {
-//         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
-//         if (isFirstOwner(c, key)) {
-//            assertIsInContainerImmortal(c, key);
-//            assert store.load(key).getValue().equals(value2);
-//         } else {
-//            assert !store.containsKey(key);
-//         }
-//      }
-//   }
-//
-//   public void testClear() throws Exception {
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//      for (int i = 0; i < 5; i++) {
-//         getOwners("k" + i)[0].put("k" + i, "value" + i);
-//         asyncWait("k" + i, PutKeyValueCommand.class, getNonOwners("k" + i));
-//      }
-//      // this will fill up L1 as well
-//      for (int i = 0; i < 5; i++) assertOnAllCachesAndOwnership("k" + i, "value" + i);
-//      for (Cache<Object, String> c : caches) assert !c.isEmpty();
-//      c1.clear();
-//      asyncWait(null, ClearCommand.class);
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//
-//      /* We only check c1 because on a shared situation, no matter where the clear is called,
-//       * it should clear the whole store regardless. Bear in mind that in the test, even though
-//       * the cache store is shared, each cache has each own cache store, that allows for checking
-//       * who execute puts, removes...etc. */
-//      CacheStore store = TestingUtil.extractComponent(c1, CacheLoaderManager.class).getCacheStore();
-//      for (int i = 0; i < 5; i++) {
-//         String key = "k" + i;
-//         assert !store.containsKey(key);
-//      }
-//   }
+   public DistSyncCacheStoreSharedTest() {
+      sync = true;
+      tx = false;
+      testRetVals = true;
+      shared = true;
+   }
+
+   public void testPutFromNonOwner() throws Exception {
+      String key = "k4", value = "value4";
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+      Cache<Object, String> nonOwner = getFirstNonOwner(key);
+      CacheStore nonOwnerStore = TestingUtil.extractComponent(nonOwner, CacheLoaderManager.class).getCacheStore();
+      assert !nonOwnerStore.containsKey(key);
+      Object retval = nonOwner.put(key, value);
+      asyncWait(key, PutKeyValueCommand.class, getSecondNonOwner(key));
+
+      Cache[] owners = getOwners(key);
+      CacheStore store = TestingUtil.extractComponent(owners[0], CacheLoaderManager.class).getCacheStore();
+      assertIsInContainerImmortal(owners[0], key);
+      assert store.containsKey(key);
+
+      for (int i = 1; i < owners.length; i++) {
+         store = TestingUtil.extractComponent(owners[i], CacheLoaderManager.class).getCacheStore();
+         assertIsInContainerImmortal(owners[i], key);
+         assert !store.containsKey(key);
+      }
+
+      for (Cache<Object, String> c : caches) {
+         store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         if (!isOwner(c, key)) {
+            assert !store.containsKey(key);
+         }
+      }
+
+      if (testRetVals) assert retval == null;
+      assertOnAllCachesAndOwnership(key, value);
+   }
+
+   public void testPutFromOwner() throws Exception {
+      String key = "k5", value = "value5";
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+      Cache[] owners = getOwners(key);
+      Object retval = owners[0].put(key, value);
+      asyncWait(key, PutKeyValueCommand.class, getNonOwners(key));
+      CacheStore store = TestingUtil.extractComponent(owners[0], CacheLoaderManager.class).getCacheStore();
+      assertIsInContainerImmortal(owners[0], key);
+      assert store.containsKey(key);
+
+      for (int i = 1; i < owners.length; i++) {
+         store = TestingUtil.extractComponent(owners[i], CacheLoaderManager.class).getCacheStore();
+         assertIsInContainerImmortal(owners[i], key);
+         assert !store.containsKey(key);
+      }
+
+      for (Cache<Object, String> c : caches) {
+         store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         if (!isOwner(c, key)) {
+            assert !store.containsKey(key);
+         }
+      }
+
+      if (testRetVals) assert retval == null;
+      assertOnAllCachesAndOwnership(key, value);
+   }
+
+
+   public void testPutAll() throws Exception {
+      String k1 = "1", v1 = "one", k2 = "2", v2 = "two", k3 = "3", v3 = "three", k4 = "4", v4 = "four";
+      String[] keys = new String[]{k1, k2, k3, k4};
+      Map<String, String> data = new HashMap<String, String>();
+      data.put(k1, v1);
+      data.put(k2, v2);
+      data.put(k3, v3);
+      data.put(k4, v4);
+
+      c1.putAll(data);
+
+      for (String key : keys) {
+         for (Cache<Object, String> c : caches) {
+            CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+            if (isFirstOwner(c, key)) {
+               assertIsInContainerImmortal(c, key);
+               assert store.containsKey(key);
+            } else {
+               assert !store.containsKey(key);
+            }
+         }
+      }
+   }
+
+   public void testRemoveFromNonOwner() throws Exception {
+      String key = "k1", value = "value";
+      initAndTest();
+
+      for (Cache<Object, String> c : caches) {
+         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         if (isFirstOwner(c, key)) {
+            assertIsInContainerImmortal(c, key);
+            assert store.load(key).getValue().equals(value);
+         } else {
+            assert !store.containsKey(key);
+         }
+      }
+
+      Object retval = getFirstNonOwner(key).remove(key);
+      asyncWait("k1", RemoveCommand.class, getSecondNonOwner("k1"));
+      if (testRetVals) assert value.equals(retval);
+      for (Cache<Object, String> c : caches) {
+         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         assert !store.containsKey(key);
+      }
+   }
+
+   public void testReplaceFromNonOwner() throws Exception {
+      String key = "k1", value = "value", value2 = "v2";
+      initAndTest();
+
+      for (Cache<Object, String> c : caches) {
+         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         if (isFirstOwner(c, key)) {
+            assertIsInContainerImmortal(c, key);
+            assert store.load(key).getValue().equals(value);
+         } else {
+            assert !store.containsKey(key);
+         }
+      }
+
+      Object retval = getFirstNonOwner(key).replace(key, value2);
+      asyncWait(key, ReplaceCommand.class, getSecondNonOwner(key));
+      if (testRetVals) assert value.equals(retval);
+      for (Cache<Object, String> c : caches) {
+         CacheStore store = TestingUtil.extractComponent(c, CacheLoaderManager.class).getCacheStore();
+         if (isFirstOwner(c, key)) {
+            assertIsInContainerImmortal(c, key);
+            assert store.load(key).getValue().equals(value2);
+         } else {
+            assert !store.containsKey(key);
+         }
+      }
+   }
+
+   public void testClear() throws Exception {
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+      for (int i = 0; i < 5; i++) {
+         getOwners("k" + i)[0].put("k" + i, "value" + i);
+         asyncWait("k" + i, PutKeyValueCommand.class, getNonOwners("k" + i));
+      }
+      // this will fill up L1 as well
+      for (int i = 0; i < 5; i++) assertOnAllCachesAndOwnership("k" + i, "value" + i);
+      for (Cache<Object, String> c : caches) assert !c.isEmpty();
+      c1.clear();
+      asyncWait(null, ClearCommand.class);
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+
+      /* We only check c1 because on a shared situation, no matter where the clear is called,
+       * it should clear the whole store regardless. Bear in mind that in the test, even though
+       * the cache store is shared, each cache has each own cache store, that allows for checking
+       * who execute puts, removes...etc. */
+      CacheStore store = TestingUtil.extractComponent(c1, CacheLoaderManager.class).getCacheStore();
+      for (int i = 0; i < 5; i++) {
+         String key = "k" + i;
+         assert !store.containsKey(key);
+      }
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistSyncFuncTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistSyncFuncTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistSyncFuncTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,185 +1,198 @@
 package org.infinispan.distribution;
 
-//@Test(groups = "functional", testName = "distribution.DistSyncFuncTest", enabled = false)
+import org.infinispan.Cache;
+import org.infinispan.commands.write.ClearCommand;
+import org.infinispan.commands.write.PutKeyValueCommand;
+import org.infinispan.commands.write.RemoveCommand;
+import org.infinispan.commands.write.ReplaceCommand;
+import org.infinispan.test.TestingUtil;
+import org.infinispan.util.ObjectDuplicator;
+import org.testng.annotations.Test;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+ at Test(groups = "functional", testName = "distribution.DistSyncFuncTest")
 public class DistSyncFuncTest extends BaseDistFunctionalTest {
 
-//   public DistSyncFuncTest() {
-//      sync = true;
-//      tx = false;
-//      testRetVals = true;
-//   }
-//
-//   public void testBasicDistribution() {
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//
-//      getOwners("k1")[0].put("k1", "value");
-//
-//      asyncWait("k1", PutKeyValueCommand.class, getNonOwners("k1"));
-//
-//      for (Cache<Object, String> c : caches) {
-//         if (isOwner(c, "k1")) {
-//            assertIsInContainerImmortal(c, "k1");
-//         } else {
-//            assertIsNotInL1(c, "k1");
-//         }
-//      }
-//
-//      // should be available everywhere!
-//      assertOnAllCachesAndOwnership("k1", "value");
-//
-//      // and should now be in L1
-//
-//      for (Cache<Object, String> c : caches) {
-//         if (isOwner(c, "k1")) {
-//            assertIsInContainerImmortal(c, "k1");
-//         } else {
-//            assertIsInL1(c, "k1");
-//         }
-//      }
-//   }
-//
-//   public void testPutFromNonOwner() {
-//      initAndTest();
-//      Cache<Object, String> nonOwner = getFirstNonOwner("k1");
-//
-//      Object retval = nonOwner.put("k1", "value2");
-//      asyncWait("k1", PutKeyValueCommand.class, getSecondNonOwner("k1"));
-//
-//      if (testRetVals) assert "value".equals(retval);
-//      assertOnAllCachesAndOwnership("k1", "value2");
-//   }
-//
-//   public void testPutIfAbsentFromNonOwner() {
-//      initAndTest();
-//      Object retval = getFirstNonOwner("k1").putIfAbsent("k1", "value2");
-//
-//      if (testRetVals) assert "value".equals(retval);
-//
-//      assertOnAllCachesAndOwnership("k1", "value");
-//
-//      c1.clear();
-//      asyncWait(null, ClearCommand.class);
-//
-//      retval = getFirstNonOwner("k1").putIfAbsent("k1", "value2");
-//      asyncWait("k1", PutKeyValueCommand.class, getSecondNonOwner("k1"));
-//      if (testRetVals) assert null == retval;
-//
-//      assertOnAllCachesAndOwnership("k1", "value2");
-//   }
-//
-//   public void testRemoveFromNonOwner() {
-//      initAndTest();
-//      Object retval = getFirstNonOwner("k1").remove("k1");
-//      asyncWait("k1", RemoveCommand.class, getSecondNonOwner("k1"));
-//      if (testRetVals) assert "value".equals(retval);
-//
-//      assertOnAllCachesAndOwnership("k1", null);
-//   }
-//
-//   public void testConditionalRemoveFromNonOwner() {
-//      initAndTest();
-//      boolean retval = getFirstNonOwner("k1").remove("k1", "value2");
-//      if (testRetVals) assert !retval : "Should not have removed entry";
-//
-//      assertOnAllCachesAndOwnership("k1", "value");
-//
-//      assert caches.get(1).get("k1").equals("value");
-//
-//      Cache<Object, String> owner = getFirstNonOwner("k1");
-//
-//      retval = owner.remove("k1", "value");
-//      asyncWait("k1", RemoveCommand.class, getSecondNonOwner("k1"));
-//      if (testRetVals) assert retval : "Should have removed entry";
-//
-//      assert caches.get(1).get("k1") == null: "expected null but received " + caches.get(1).get("k1");
-//      assertOnAllCachesAndOwnership("k1", null);
-//   }
-//
-//   public void testReplaceFromNonOwner() {
-//      initAndTest();
-//      Object retval = getFirstNonOwner("k1").replace("k1", "value2");
-//      if (testRetVals) assert "value".equals(retval);
-//
-//      asyncWait("k1", ReplaceCommand.class, getSecondNonOwner("k1"));
-//
-//      assertOnAllCachesAndOwnership("k1", "value2");
-//
-//      c1.clear();
-//      asyncWait(null, ClearCommand.class);
-//
-//      retval = getFirstNonOwner("k1").replace("k1", "value2");
-//      if (testRetVals) assert retval == null;
-//
-//      assertOnAllCachesAndOwnership("k1", null);
-//   }
-//
-//   public void testConditionalReplaceFromNonOwner() {
-//      initAndTest();
-//      Cache<Object, String> nonOwner = getFirstNonOwner("k1");
-//      boolean retval = nonOwner.replace("k1", "valueX", "value2");
-//      if (testRetVals) assert !retval : "Should not have replaced";
-//
-//      assertOnAllCachesAndOwnership("k1", "value");
-//
-//      assert !nonOwner.getAdvancedCache().getComponentRegistry().getComponent(DistributionManager.class).isLocal("k1");
-//      retval = nonOwner.replace("k1", "value", "value2");
-//      asyncWait("k1", ReplaceCommand.class, getSecondNonOwner("k1"));
-//      if (testRetVals) assert retval : "Should have replaced";
-//
-//      assertOnAllCachesAndOwnership("k1", "value2");
-//   }
-//
-//   public void testClear() {
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//
-//      for (int i = 0; i < 10; i++) {
-//         getOwners("k" + i)[0].put("k" + i, "value" + i);
-//         asyncWait("k" + i, PutKeyValueCommand.class, getNonOwners("k" + i));
-//      }
-//
-//      // this will fill up L1 as well
-//      for (int i = 0; i < 10; i++) assertOnAllCachesAndOwnership("k" + i, "value" + i);
-//
-//      for (Cache<Object, String> c : caches) assert !c.isEmpty();
-//
-//      c1.clear();
-//      asyncWait(null, ClearCommand.class);
-//
-//      for (Cache<Object, String> c : caches) assert c.isEmpty();
-//   }
-//
-//   public void testKeyValueEntryCollections() {
-//      c1.put("1", "one");
-//      asyncWait("1", PutKeyValueCommand.class, getNonOwnersExcludingSelf("1", addressOf(c1)));
-//      c2.put("2", "two");
-//      asyncWait("2", PutKeyValueCommand.class, getNonOwnersExcludingSelf("2", addressOf(c2)));
-//      c3.put("3", "three");
-//      asyncWait("3", PutKeyValueCommand.class, getNonOwnersExcludingSelf("3", addressOf(c3)));
-//      c4.put("4", "four");
-//      asyncWait("4", PutKeyValueCommand.class, getNonOwnersExcludingSelf("4", addressOf(c4)));
-//
-//      for (Cache c : caches) {
-//         Set expKeys = TestingUtil.getInternalKeys(c);
-//         Collection expValues = TestingUtil.getInternalValues(c);
-//
-//         Set expKeyEntries = ObjectDuplicator.duplicateSet(expKeys);
-//         Collection expValueEntries = ObjectDuplicator.duplicateCollection(expValues);
-//
-//         Set keys = c.keySet();
-//         for (Object key : keys) assert expKeys.remove(key);
-//         assert expKeys.isEmpty() : "Did not see keys " + expKeys + " in iterator!";
-//
-//         Collection values = c.values();
-//         for (Object value : values) assert expValues.remove(value);
-//         assert expValues.isEmpty() : "Did not see keys " + expValues + " in iterator!";
-//
-//         Set<Map.Entry> entries = c.entrySet();
-//         for (Map.Entry entry : entries) {
-//            assert expKeyEntries.remove(entry.getKey());
-//            assert expValueEntries.remove(entry.getValue());
-//         }
-//         assert expKeyEntries.isEmpty() : "Did not see keys " + expKeyEntries + " in iterator!";
-//         assert expValueEntries.isEmpty() : "Did not see keys " + expValueEntries + " in iterator!";
-//      }
-//   }
+   public DistSyncFuncTest() {
+      sync = true;
+      tx = false;
+      testRetVals = true;
+   }
+
+   public void testBasicDistribution() {
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+
+      getOwners("k1")[0].put("k1", "value");
+
+      asyncWait("k1", PutKeyValueCommand.class, getNonOwners("k1"));
+
+      for (Cache<Object, String> c : caches) {
+         if (isOwner(c, "k1")) {
+            assertIsInContainerImmortal(c, "k1");
+         } else {
+            assertIsNotInL1(c, "k1");
+         }
+      }
+
+      // should be available everywhere!
+      assertOnAllCachesAndOwnership("k1", "value");
+
+      // and should now be in L1
+
+      for (Cache<Object, String> c : caches) {
+         if (isOwner(c, "k1")) {
+            assertIsInContainerImmortal(c, "k1");
+         } else {
+            assertIsInL1(c, "k1");
+         }
+      }
+   }
+
+   public void testPutFromNonOwner() {
+      initAndTest();
+      Cache<Object, String> nonOwner = getFirstNonOwner("k1");
+
+      Object retval = nonOwner.put("k1", "value2");
+      asyncWait("k1", PutKeyValueCommand.class, getSecondNonOwner("k1"));
+
+      if (testRetVals) assert "value".equals(retval);
+      assertOnAllCachesAndOwnership("k1", "value2");
+   }
+
+   public void testPutIfAbsentFromNonOwner() {
+      initAndTest();
+      Object retval = getFirstNonOwner("k1").putIfAbsent("k1", "value2");
+
+      if (testRetVals) assert "value".equals(retval);
+
+      assertOnAllCachesAndOwnership("k1", "value");
+
+      c1.clear();
+      asyncWait(null, ClearCommand.class);
+
+      retval = getFirstNonOwner("k1").putIfAbsent("k1", "value2");
+      asyncWait("k1", PutKeyValueCommand.class, getSecondNonOwner("k1"));
+      if (testRetVals) assert null == retval;
+
+      assertOnAllCachesAndOwnership("k1", "value2");
+   }
+
+   public void testRemoveFromNonOwner() {
+      initAndTest();
+      Object retval = getFirstNonOwner("k1").remove("k1");
+      asyncWait("k1", RemoveCommand.class, getSecondNonOwner("k1"));
+      if (testRetVals) assert "value".equals(retval);
+
+      assertOnAllCachesAndOwnership("k1", null);
+   }
+
+   public void testConditionalRemoveFromNonOwner() {
+      initAndTest();
+      boolean retval = getFirstNonOwner("k1").remove("k1", "value2");
+      if (testRetVals) assert !retval : "Should not have removed entry";
+
+      assertOnAllCachesAndOwnership("k1", "value");
+
+      assert caches.get(1).get("k1").equals("value");
+
+      Cache<Object, String> owner = getFirstNonOwner("k1");
+
+      retval = owner.remove("k1", "value");
+      asyncWait("k1", RemoveCommand.class, getSecondNonOwner("k1"));
+      if (testRetVals) assert retval : "Should have removed entry";
+
+      assert caches.get(1).get("k1") == null : "expected null but received " + caches.get(1).get("k1");
+      assertOnAllCachesAndOwnership("k1", null);
+   }
+
+   public void testReplaceFromNonOwner() {
+      initAndTest();
+      Object retval = getFirstNonOwner("k1").replace("k1", "value2");
+      if (testRetVals) assert "value".equals(retval);
+
+      asyncWait("k1", ReplaceCommand.class, getSecondNonOwner("k1"));
+
+      assertOnAllCachesAndOwnership("k1", "value2");
+
+      c1.clear();
+      asyncWait(null, ClearCommand.class);
+
+      retval = getFirstNonOwner("k1").replace("k1", "value2");
+      if (testRetVals) assert retval == null;
+
+      assertOnAllCachesAndOwnership("k1", null);
+   }
+
+   public void testConditionalReplaceFromNonOwner() {
+      initAndTest();
+      Cache<Object, String> nonOwner = getFirstNonOwner("k1");
+      boolean retval = nonOwner.replace("k1", "valueX", "value2");
+      if (testRetVals) assert !retval : "Should not have replaced";
+
+      assertOnAllCachesAndOwnership("k1", "value");
+
+      assert !nonOwner.getAdvancedCache().getComponentRegistry().getComponent(DistributionManager.class).isLocal("k1");
+      retval = nonOwner.replace("k1", "value", "value2");
+      asyncWait("k1", ReplaceCommand.class, getSecondNonOwner("k1"));
+      if (testRetVals) assert retval : "Should have replaced";
+
+      assertOnAllCachesAndOwnership("k1", "value2");
+   }
+
+   public void testClear() {
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+
+      for (int i = 0; i < 10; i++) {
+         getOwners("k" + i)[0].put("k" + i, "value" + i);
+         asyncWait("k" + i, PutKeyValueCommand.class, getNonOwners("k" + i));
+      }
+
+      // this will fill up L1 as well
+      for (int i = 0; i < 10; i++) assertOnAllCachesAndOwnership("k" + i, "value" + i);
+
+      for (Cache<Object, String> c : caches) assert !c.isEmpty();
+
+      c1.clear();
+      asyncWait(null, ClearCommand.class);
+
+      for (Cache<Object, String> c : caches) assert c.isEmpty();
+   }
+
+   public void testKeyValueEntryCollections() {
+      c1.put("1", "one");
+      asyncWait("1", PutKeyValueCommand.class, getNonOwnersExcludingSelf("1", addressOf(c1)));
+      c2.put("2", "two");
+      asyncWait("2", PutKeyValueCommand.class, getNonOwnersExcludingSelf("2", addressOf(c2)));
+      c3.put("3", "three");
+      asyncWait("3", PutKeyValueCommand.class, getNonOwnersExcludingSelf("3", addressOf(c3)));
+      c4.put("4", "four");
+      asyncWait("4", PutKeyValueCommand.class, getNonOwnersExcludingSelf("4", addressOf(c4)));
+
+      for (Cache c : caches) {
+         Set expKeys = TestingUtil.getInternalKeys(c);
+         Collection expValues = TestingUtil.getInternalValues(c);
+
+         Set expKeyEntries = ObjectDuplicator.duplicateSet(expKeys);
+         Collection expValueEntries = ObjectDuplicator.duplicateCollection(expValues);
+
+         Set keys = c.keySet();
+         for (Object key : keys) assert expKeys.remove(key);
+         assert expKeys.isEmpty() : "Did not see keys " + expKeys + " in iterator!";
+
+         Collection values = c.values();
+         for (Object value : values) assert expValues.remove(value);
+         assert expValues.isEmpty() : "Did not see keys " + expValues + " in iterator!";
+
+         Set<Map.Entry> entries = c.entrySet();
+         for (Map.Entry entry : entries) {
+            assert expKeyEntries.remove(entry.getKey());
+            assert expValueEntries.remove(entry.getValue());
+         }
+         assert expKeyEntries.isEmpty() : "Did not see keys " + expKeyEntries + " in iterator!";
+         assert expValueEntries.isEmpty() : "Did not see keys " + expValueEntries + " in iterator!";
+      }
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistSyncTxFuncTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistSyncTxFuncTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistSyncTxFuncTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,328 +1,336 @@
 package org.infinispan.distribution;
 
-//@Test(groups = "functional", testName = "distribution.DistSyncTxFuncTest", enabled = false)
+import org.infinispan.Cache;
+import org.infinispan.commands.write.PutKeyValueCommand;
+import org.infinispan.test.TestingUtil;
+import org.infinispan.util.concurrent.locks.LockManager;
+import org.testng.annotations.Test;
+
+import javax.transaction.TransactionManager;
+
+ at Test(groups = "functional", testName = "distribution.DistSyncTxFuncTest")
 public class DistSyncTxFuncTest extends BaseDistFunctionalTest {
-//   public DistSyncTxFuncTest() {
-//      sync = true;
-//      tx = true;
-//      testRetVals = true;
-//      cleanup = CleanupPhase.AFTER_METHOD; // ensure any stale TXs are wiped
-//   }
-//
-//   protected void asyncTxWait(Object... keys) {
-//      // no op.  Meant to be overridden
-//   }
-//
-//   protected void init(MagicKey k1, MagicKey k2) {
-//      // neither key maps on to c4
-//      c2.put(k1, "value1");
-//      asyncWait(k1, PutKeyValueCommand.class, c1, c3, c4);
-//
-//      c2.put(k2, "value2");
-//      asyncWait(k2, PutKeyValueCommand.class, c1, c3, c4);
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsInContainerImmortal(c2, k2);
-//      assertIsInContainerImmortal(c3, k2);
-//
-//      assertIsNotInL1(c4, k1);
-//      assertIsNotInL1(c4, k2);
-//      assertIsNotInL1(c1, k2);
-//      assertIsNotInL1(c3, k1);
-//   }
-//
-//   public void testTransactionsSpanningKeysCommit() throws Exception {
-   // we need 2 keys that reside on different caches...
-//      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
-//      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
-//
-//      init(k1, k2);
-//
-//      // now test a transaction that spans both keys.
-//      TransactionManager tm4 = getTransactionManager(c4);
-//      asserLocked(c3, false, k1);
-//      tm4.begin();
-//      c4.put(k1, "new_value1");
-//      c4.put(k2, "new_value2");
-//      tm4.commit();
-//
-//      asyncTxWait("new_value1","new_value2");
-//
-//      asserLocked(c3, false, k1);
-//      asserLocked(c3, false, k2);
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsInContainerImmortal(c2, k2);
-//      assertIsInContainerImmortal(c3, k2);
-//
-//      assertIsInL1(c4, k1);
-//      assertIsInL1(c4, k2);
-//      assertIsNotInL1(c1, k2);
-//      assertIsNotInL1(c3, k1);
-//
-//      asserLocked(c4, false, k1, k2);
-//      asserLocked(c3, false, k1);
-//      asserLocked(c3, false, k2);
-//      asserLocked(c1, false, k1, k2);
-//      asserLocked(c2, false, k1, k2);
-//      checkOwnership(k1, k2, "new_value1", "new_value2");
-//   }
-//
-//   void asserLocked(Cache c, boolean isLocked, Object... keys) {
-//      LockManager lm = TestingUtil.extractComponent(c, LockManager.class);
-//      for (Object key : keys) {
-//         assert isLocked == lm.isLocked(key) : " expecting key '" + key + "' to be "  + (isLocked ?  " locked " :
-//               "not locked + \n Lock owner is:" + lm.getOwner(key));
-//      }
-//   }
-//
-//   protected void checkOwnership(MagicKey k1, MagicKey k2, String v1, String v2) {
-//      assertOnAllCachesAndOwnership(k1, v1);
-//      assertOnAllCachesAndOwnership(k2, v2);
-//
-//      assertIsInL1(c4, k1);
-//      assertIsInL1(c4, k2);
-//      assertIsInL1(c1, k2);
-//      assertIsInL1(c3, k1);
-//   }
-//
-//   public void testTransactionsSpanningKeysRollback() throws Exception {
-//      // we need 2 keys that reside on different caches...
-//      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
-//      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
-//
-//      init(k1, k2);
-//
-//      // now test a transaction that spans both keys.
-//      TransactionManager tm4 = getTransactionManager(c4);
-//      tm4.begin();
-//      c4.put(k1, "new_value1");
-//      c4.put(k2, "new_value2");
-//      tm4.rollback();
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsInContainerImmortal(c2, k2);
-//      assertIsInContainerImmortal(c3, k2);
-//
-//      assertIsNotInL1(c4, k1);
-//      assertIsNotInL1(c4, k2);
-//      assertIsNotInL1(c1, k2);
-//      assertIsNotInL1(c3, k1);
-//
-//      checkOwnership(k1, k2, "value1", "value2");
-//   }
-//
-//   public void testPutFromNonOwner() throws Exception {
-//      // we need 2 keys that reside on different caches...
-//      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
-//      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
-//
-//      init(k1, k2);
-//
-//      TransactionManager tm4 = getTransactionManager(c4);
-//      tm4.begin();
-//      Object ret = c4.put(k1, "new_value");
-//      if (testRetVals) assert "value1".equals(ret);
-//      ret = c4.put(k2, "new_value");
-//      if (testRetVals) assert "value2".equals(ret);
-//      tm4.rollback();
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsInContainerImmortal(c2, k2);
-//      assertIsInContainerImmortal(c3, k2);
-//
-//      assertIsNotInL1(c4, k1);
-//      assertIsNotInL1(c4, k2);
-//      assertIsNotInL1(c1, k2);
-//      assertIsNotInL1(c3, k1);
-//
-//      checkOwnership(k1, k2, "value1", "value2");
-//   }
-//
-//   public void testPutIfAbsentFromNonOwner() throws Exception {
-//      // we need 2 keys that reside on different caches...
-//      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
-//      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
-//
-//      init(k1, k2);
-//
-//      TransactionManager tm4 = getTransactionManager(c4);
-//      LockManager lockManager4 = TestingUtil.extractComponent(c4, LockManager.class);
-//
-//      tm4.begin();
-//      Object ret = c4.putIfAbsent(k1, "new_value");
-//      if (testRetVals) assert "value1".equals(ret) : "Was expecting value1 but was " + ret;
-//      ret = c4.putIfAbsent(k2, "new_value");
-//      if (testRetVals) assert "value2".equals(ret) : "Was expecting value2 but was " + ret;
-//
-//      assert c4.get(k1).equals("value1");
-//      assert c4.get(k2).equals("value2");
-//
-//      assert lockManager4.isLocked(k1);
-//      assert lockManager4.isLocked(k2);
-//
-//      tm4.rollback();
-//
-//      assert !lockManager4.isLocked(k1);
-//      assert !lockManager4.isLocked(k2);
-//
-//      assert c2.get(k1).equals("value1");
-//      assert c2.get(k2).equals("value2");
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsInContainerImmortal(c2, k2);
-//      assertIsInContainerImmortal(c3, k2);
-//
-//      assertIsNotInL1(c4, k1);
-//      assertIsNotInL1(c4, k2);
-//      assertIsNotInL1(c1, k2);
-//      assertIsNotInL1(c3, k1);
-//
-//      checkOwnership(k1, k2, "value1", "value2");
-//   }
-//
-//   public void testRemoveFromNonOwner() throws Exception {
-//      // we need 2 keys that reside on different caches...
-//      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
-//      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
-//
-//      init(k1, k2);
-//
-//      TransactionManager tm4 = getTransactionManager(c4);
-//      tm4.begin();
-//      Object ret = c4.remove(k1);
-//      if (testRetVals) assert "value1".equals(ret);
-//      ret = c4.remove(k2);
-//      if (testRetVals) assert "value2".equals(ret);
-//
-//      assert !c4.containsKey(k1);
-//      assert !c4.containsKey(k2);
-//      tm4.rollback();
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsInContainerImmortal(c2, k2);
-//      assertIsInContainerImmortal(c3, k2);
-//
-//      assertIsNotInL1(c4, k1);
-//      assertIsNotInL1(c4, k2);
-//      assertIsNotInL1(c1, k2);
-//      assertIsNotInL1(c3, k1);
-//
-//      checkOwnership(k1, k2, "value1", "value2");
-//   }
-//
-//   public void testConditionalRemoveFromNonOwner() throws Exception {
-//      // we need 2 keys that reside on different caches...
-//      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
-//      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
-//
-//      init(k1, k2);
-//
-//      TransactionManager tm4 = getTransactionManager(c4);
-//      tm4.begin();
-//      boolean ret = c4.remove(k1, "valueX");
-//      if (testRetVals) assert !ret;
-//      ret = c4.remove(k2, "valueX");
-//      if (testRetVals) assert !ret;
-//
-//      assert c4.containsKey(k1);
-//      assert c4.containsKey(k2);
-//
-//      ret = c4.remove(k1, "value1");
-//      if (testRetVals) assert ret;
-//      ret = c4.remove(k2, "value2");
-//      if (testRetVals) assert ret;
-//
-//      assert !c4.containsKey(k1);
-//      assert !c4.containsKey(k2);
-//      tm4.rollback();
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsInContainerImmortal(c2, k2);
-//      assertIsInContainerImmortal(c3, k2);
-//
-//      assertIsNotInL1(c4, k1);
-//      assertIsNotInL1(c4, k2);
-//      assertIsNotInL1(c1, k2);
-//      assertIsNotInL1(c3, k1);
-//
-//      checkOwnership(k1, k2, "value1", "value2");
-//   }
-//
-//   public void testReplaceFromNonOwner() throws Exception {
-//      // we need 2 keys that reside on different caches...
-//      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
-//      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
-//
-//      init(k1, k2);
-//
-//      TransactionManager tm4 = getTransactionManager(c4);
-//      tm4.begin();
-//      Object ret = c4.replace(k1, "new_value");
-//      if (testRetVals) assert "value1".equals(ret);
-//      ret = c4.replace(k2, "new_value");
-//      if (testRetVals) assert "value2".equals(ret);
-//
-//      assert "new_value".equals(c4.get(k1));
-//      assert "new_value".equals(c4.get(k2));
-//      tm4.rollback();
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsInContainerImmortal(c2, k2);
-//      assertIsInContainerImmortal(c3, k2);
-//
-//      assertIsNotInL1(c4, k1);
-//      assertIsNotInL1(c4, k2);
-//      assertIsNotInL1(c1, k2);
-//      assertIsNotInL1(c3, k1);
-//
-//      checkOwnership(k1, k2, "value1", "value2");
-//   }
-//
-//   public void testConditionalReplaceFromNonOwner() throws Exception {
-//      // we need 2 keys that reside on different caches...
-//      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
-//      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
-//
-//      init(k1, k2);
-//
-//      TransactionManager tm4 = getTransactionManager(c4);
-//      tm4.begin();
-//      boolean ret = c4.replace(k1, "valueX", "new_value");
-//      if (testRetVals) assert !ret;
-//      ret = c4.replace(k2, "valueX", "new_value");
-//      if (testRetVals) assert !ret;
-//
-//      assert "value1".equals(c4.get(k1));
-//      assert "value2".equals(c4.get(k2));
-//
-//      ret = c4.replace(k1, "value1", "new_value");
-//      if (testRetVals) assert ret;
-//      ret = c4.replace(k2, "value2", "new_value");
-//      if (testRetVals) assert ret;
-//
-//      assert "new_value".equals(c4.get(k1));
-//      assert "new_value".equals(c4.get(k2));
-//      tm4.rollback();
-//
-//      assertIsInContainerImmortal(c1, k1);
-//      assertIsInContainerImmortal(c2, k1);
-//      assertIsInContainerImmortal(c2, k2);
-//      assertIsInContainerImmortal(c3, k2);
-//
-//      assertIsNotInL1(c4, k1);
-//      assertIsNotInL1(c4, k2);
-//      assertIsNotInL1(c1, k2);
-//      assertIsNotInL1(c3, k1);
-//
-//      checkOwnership(k1, k2, "value1", "value2");
-//   }
+   public DistSyncTxFuncTest() {
+      sync = true;
+      tx = true;
+      testRetVals = true;
+      cleanup = CleanupPhase.AFTER_METHOD; // ensure any stale TXs are wiped
+   }
+
+   protected void asyncTxWait(Object... keys) {
+      // no op.  Meant to be overridden
+   }
+
+   protected void init(MagicKey k1, MagicKey k2) {
+      // neither key maps on to c4
+      c2.put(k1, "value1");
+      asyncWait(k1, PutKeyValueCommand.class, c1, c3, c4);
+
+      c2.put(k2, "value2");
+      asyncWait(k2, PutKeyValueCommand.class, c1, c3, c4);
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsInContainerImmortal(c2, k2);
+      assertIsInContainerImmortal(c3, k2);
+
+      assertIsNotInL1(c4, k1);
+      assertIsNotInL1(c4, k2);
+      assertIsNotInL1(c1, k2);
+      assertIsNotInL1(c3, k1);
+   }
+
+   public void testTransactionsSpanningKeysCommit() throws Exception {
+//    we need 2 keys that reside on different caches...
+      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
+      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
+
+      init(k1, k2);
+
+      // now test a transaction that spans both keys.
+      TransactionManager tm4 = getTransactionManager(c4);
+      asserLocked(c3, false, k1);
+      tm4.begin();
+      c4.put(k1, "new_value1");
+      c4.put(k2, "new_value2");
+      tm4.commit();
+
+      asyncTxWait("new_value1", "new_value2");
+
+      asserLocked(c3, false, k1);
+      asserLocked(c3, false, k2);
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsInContainerImmortal(c2, k2);
+      assertIsInContainerImmortal(c3, k2);
+
+      assertIsInL1(c4, k1);
+      assertIsInL1(c4, k2);
+      assertIsNotInL1(c1, k2);
+      assertIsNotInL1(c3, k1);
+
+      asserLocked(c4, false, k1, k2);
+      asserLocked(c3, false, k1);
+      asserLocked(c3, false, k2);
+      asserLocked(c1, false, k1, k2);
+      asserLocked(c2, false, k1, k2);
+      checkOwnership(k1, k2, "new_value1", "new_value2");
+   }
+
+   void asserLocked(Cache c, boolean isLocked, Object... keys) {
+      LockManager lm = TestingUtil.extractComponent(c, LockManager.class);
+      for (Object key : keys) {
+         assert isLocked == lm.isLocked(key) : " expecting key '" + key + "' to be " + (isLocked ? " locked " :
+               "not locked + \n Lock owner is:" + lm.getOwner(key));
+      }
+   }
+
+   protected void checkOwnership(MagicKey k1, MagicKey k2, String v1, String v2) {
+      assertOnAllCachesAndOwnership(k1, v1);
+      assertOnAllCachesAndOwnership(k2, v2);
+
+      assertIsInL1(c4, k1);
+      assertIsInL1(c4, k2);
+      assertIsInL1(c1, k2);
+      assertIsInL1(c3, k1);
+   }
+
+   public void testTransactionsSpanningKeysRollback() throws Exception {
+      // we need 2 keys that reside on different caches...
+      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
+      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
+
+      init(k1, k2);
+
+      // now test a transaction that spans both keys.
+      TransactionManager tm4 = getTransactionManager(c4);
+      tm4.begin();
+      c4.put(k1, "new_value1");
+      c4.put(k2, "new_value2");
+      tm4.rollback();
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsInContainerImmortal(c2, k2);
+      assertIsInContainerImmortal(c3, k2);
+
+      assertIsNotInL1(c4, k1);
+      assertIsNotInL1(c4, k2);
+      assertIsNotInL1(c1, k2);
+      assertIsNotInL1(c3, k1);
+
+      checkOwnership(k1, k2, "value1", "value2");
+   }
+
+   public void testPutFromNonOwner() throws Exception {
+      // we need 2 keys that reside on different caches...
+      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
+      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
+
+      init(k1, k2);
+
+      TransactionManager tm4 = getTransactionManager(c4);
+      tm4.begin();
+      Object ret = c4.put(k1, "new_value");
+      if (testRetVals) assert "value1".equals(ret);
+      ret = c4.put(k2, "new_value");
+      if (testRetVals) assert "value2".equals(ret);
+      tm4.rollback();
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsInContainerImmortal(c2, k2);
+      assertIsInContainerImmortal(c3, k2);
+
+      assertIsNotInL1(c4, k1);
+      assertIsNotInL1(c4, k2);
+      assertIsNotInL1(c1, k2);
+      assertIsNotInL1(c3, k1);
+
+      checkOwnership(k1, k2, "value1", "value2");
+   }
+
+   public void testPutIfAbsentFromNonOwner() throws Exception {
+      // we need 2 keys that reside on different caches...
+      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
+      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
+
+      init(k1, k2);
+
+      TransactionManager tm4 = getTransactionManager(c4);
+      LockManager lockManager4 = TestingUtil.extractComponent(c4, LockManager.class);
+
+      tm4.begin();
+      Object ret = c4.putIfAbsent(k1, "new_value");
+      if (testRetVals) assert "value1".equals(ret) : "Was expecting value1 but was " + ret;
+      ret = c4.putIfAbsent(k2, "new_value");
+      if (testRetVals) assert "value2".equals(ret) : "Was expecting value2 but was " + ret;
+
+      assert c4.get(k1).equals("value1");
+      assert c4.get(k2).equals("value2");
+
+      assert lockManager4.isLocked(k1);
+      assert lockManager4.isLocked(k2);
+
+      tm4.rollback();
+
+      assert !lockManager4.isLocked(k1);
+      assert !lockManager4.isLocked(k2);
+
+      assert c2.get(k1).equals("value1");
+      assert c2.get(k2).equals("value2");
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsInContainerImmortal(c2, k2);
+      assertIsInContainerImmortal(c3, k2);
+
+      assertIsNotInL1(c4, k1);
+      assertIsNotInL1(c4, k2);
+      assertIsNotInL1(c1, k2);
+      assertIsNotInL1(c3, k1);
+
+      checkOwnership(k1, k2, "value1", "value2");
+   }
+
+   public void testRemoveFromNonOwner() throws Exception {
+      // we need 2 keys that reside on different caches...
+      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
+      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
+
+      init(k1, k2);
+
+      TransactionManager tm4 = getTransactionManager(c4);
+      tm4.begin();
+      Object ret = c4.remove(k1);
+      if (testRetVals) assert "value1".equals(ret);
+      ret = c4.remove(k2);
+      if (testRetVals) assert "value2".equals(ret);
+
+      assert !c4.containsKey(k1);
+      assert !c4.containsKey(k2);
+      tm4.rollback();
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsInContainerImmortal(c2, k2);
+      assertIsInContainerImmortal(c3, k2);
+
+      assertIsNotInL1(c4, k1);
+      assertIsNotInL1(c4, k2);
+      assertIsNotInL1(c1, k2);
+      assertIsNotInL1(c3, k1);
+
+      checkOwnership(k1, k2, "value1", "value2");
+   }
+
+   public void testConditionalRemoveFromNonOwner() throws Exception {
+      // we need 2 keys that reside on different caches...
+      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
+      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
+
+      init(k1, k2);
+
+      TransactionManager tm4 = getTransactionManager(c4);
+      tm4.begin();
+      boolean ret = c4.remove(k1, "valueX");
+      if (testRetVals) assert !ret;
+      ret = c4.remove(k2, "valueX");
+      if (testRetVals) assert !ret;
+
+      assert c4.containsKey(k1);
+      assert c4.containsKey(k2);
+
+      ret = c4.remove(k1, "value1");
+      if (testRetVals) assert ret;
+      ret = c4.remove(k2, "value2");
+      if (testRetVals) assert ret;
+
+      assert !c4.containsKey(k1);
+      assert !c4.containsKey(k2);
+      tm4.rollback();
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsInContainerImmortal(c2, k2);
+      assertIsInContainerImmortal(c3, k2);
+
+      assertIsNotInL1(c4, k1);
+      assertIsNotInL1(c4, k2);
+      assertIsNotInL1(c1, k2);
+      assertIsNotInL1(c3, k1);
+
+      checkOwnership(k1, k2, "value1", "value2");
+   }
+
+   public void testReplaceFromNonOwner() throws Exception {
+      // we need 2 keys that reside on different caches...
+      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
+      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
+
+      init(k1, k2);
+
+      TransactionManager tm4 = getTransactionManager(c4);
+      tm4.begin();
+      Object ret = c4.replace(k1, "new_value");
+      if (testRetVals) assert "value1".equals(ret);
+      ret = c4.replace(k2, "new_value");
+      if (testRetVals) assert "value2".equals(ret);
+
+      assert "new_value".equals(c4.get(k1));
+      assert "new_value".equals(c4.get(k2));
+      tm4.rollback();
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsInContainerImmortal(c2, k2);
+      assertIsInContainerImmortal(c3, k2);
+
+      assertIsNotInL1(c4, k1);
+      assertIsNotInL1(c4, k2);
+      assertIsNotInL1(c1, k2);
+      assertIsNotInL1(c3, k1);
+
+      checkOwnership(k1, k2, "value1", "value2");
+   }
+
+   public void testConditionalReplaceFromNonOwner() throws Exception {
+      // we need 2 keys that reside on different caches...
+      MagicKey k1 = new MagicKey(c1); // maps on to c1 and c2
+      MagicKey k2 = new MagicKey(c2); // maps on to c2 and c3
+
+      init(k1, k2);
+
+      TransactionManager tm4 = getTransactionManager(c4);
+      tm4.begin();
+      boolean ret = c4.replace(k1, "valueX", "new_value");
+      if (testRetVals) assert !ret;
+      ret = c4.replace(k2, "valueX", "new_value");
+      if (testRetVals) assert !ret;
+
+      assert "value1".equals(c4.get(k1));
+      assert "value2".equals(c4.get(k2));
+
+      ret = c4.replace(k1, "value1", "new_value");
+      if (testRetVals) assert ret;
+      ret = c4.replace(k2, "value2", "new_value");
+      if (testRetVals) assert ret;
+
+      assert "new_value".equals(c4.get(k1));
+      assert "new_value".equals(c4.get(k2));
+      tm4.rollback();
+
+      assertIsInContainerImmortal(c1, k1);
+      assertIsInContainerImmortal(c2, k1);
+      assertIsInContainerImmortal(c2, k2);
+      assertIsInContainerImmortal(c3, k2);
+
+      assertIsNotInL1(c4, k1);
+      assertIsNotInL1(c4, k2);
+      assertIsNotInL1(c1, k2);
+      assertIsNotInL1(c3, k1);
+
+      checkOwnership(k1, k2, "value1", "value2");
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistSyncTxUnsafeFuncTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistSyncTxUnsafeFuncTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistSyncTxUnsafeFuncTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,11 +1,13 @@
 package org.infinispan.distribution;
 
-//@Test(groups = "functional", testName = "distribution.DistSyncTxUnsafeFuncTest", enabled = false)
+import org.testng.annotations.Test;
+
+ at Test(groups = "functional", testName = "distribution.DistSyncTxUnsafeFuncTest")
 public class DistSyncTxUnsafeFuncTest extends DistSyncTxFuncTest {
-//   public DistSyncTxUnsafeFuncTest() {
-//      sync = true;
-//      tx = true;
-//      testRetVals = false;
-//      cleanup = CleanupPhase.AFTER_METHOD; // ensure any stale TXs are wiped
-//   }
+   public DistSyncTxUnsafeFuncTest() {
+      sync = true;
+      tx = true;
+      testRetVals = false;
+      cleanup = CleanupPhase.AFTER_METHOD; // ensure any stale TXs are wiped
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/DistSyncUnsafeFuncTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/DistSyncUnsafeFuncTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/DistSyncUnsafeFuncTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,10 +1,12 @@
 package org.infinispan.distribution;
 
-//@Test(groups = "functional", testName = "distribution.DistSyncUnsafeFuncTest", enabled = false)
+import org.testng.annotations.Test;
+
+ at Test(groups = "functional", testName = "distribution.DistSyncUnsafeFuncTest")
 public class DistSyncUnsafeFuncTest extends DistSyncFuncTest {
-//   public DistSyncUnsafeFuncTest() {
-//      sync = true;
-//      tx = false;
-//      testRetVals = false;
-//   }
+   public DistSyncUnsafeFuncTest() {
+      sync = true;
+      tx = false;
+      testRetVals = false;
+   }
 }

Modified: trunk/core/src/test/java/org/infinispan/distribution/MagicKeyTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/MagicKeyTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/distribution/MagicKeyTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -1,30 +1,32 @@
 package org.infinispan.distribution;
 
-//@Test(groups = "unit", testName = "distribution.MagicKeyTest", enabled = false)
+import org.testng.annotations.Test;
+
+ at Test(groups = "unit", testName = "distribution.MagicKeyTest")
 public class MagicKeyTest extends BaseDistFunctionalTest {
-//   public void testMagicKeys() {
-//      BaseDistFunctionalTest.MagicKey k1 = new BaseDistFunctionalTest.MagicKey(c1);
-//      assert getDistributionManager(c1).isLocal(k1);
-//      assert getDistributionManager(c2).isLocal(k1);
-//      assert !getDistributionManager(c3).isLocal(k1);
-//      assert !getDistributionManager(c4).isLocal(k1);
-//
-//      BaseDistFunctionalTest.MagicKey k2 = new BaseDistFunctionalTest.MagicKey(c2);
-//      assert !getDistributionManager(c1).isLocal(k2);
-//      assert getDistributionManager(c2).isLocal(k2);
-//      assert getDistributionManager(c3).isLocal(k2);
-//      assert !getDistributionManager(c4).isLocal(k2);
-//
-//      BaseDistFunctionalTest.MagicKey k3 = new BaseDistFunctionalTest.MagicKey(c3);
-//      assert !getDistributionManager(c1).isLocal(k3);
-//      assert !getDistributionManager(c2).isLocal(k3);
-//      assert getDistributionManager(c3).isLocal(k3);
-//      assert getDistributionManager(c4).isLocal(k3);
-//
-//      BaseDistFunctionalTest.MagicKey k4 = new BaseDistFunctionalTest.MagicKey(c4);
-//      assert getDistributionManager(c1).isLocal(k4);
-//      assert !getDistributionManager(c2).isLocal(k4);
-//      assert !getDistributionManager(c3).isLocal(k4);
-//      assert getDistributionManager(c4).isLocal(k4);
-//   }
+   public void testMagicKeys() {
+      BaseDistFunctionalTest.MagicKey k1 = new BaseDistFunctionalTest.MagicKey(c1);
+      assert getDistributionManager(c1).isLocal(k1);
+      assert getDistributionManager(c2).isLocal(k1);
+      assert !getDistributionManager(c3).isLocal(k1);
+      assert !getDistributionManager(c4).isLocal(k1);
+
+      BaseDistFunctionalTest.MagicKey k2 = new BaseDistFunctionalTest.MagicKey(c2);
+      assert !getDistributionManager(c1).isLocal(k2);
+      assert getDistributionManager(c2).isLocal(k2);
+      assert getDistributionManager(c3).isLocal(k2);
+      assert !getDistributionManager(c4).isLocal(k2);
+
+      BaseDistFunctionalTest.MagicKey k3 = new BaseDistFunctionalTest.MagicKey(c3);
+      assert !getDistributionManager(c1).isLocal(k3);
+      assert !getDistributionManager(c2).isLocal(k3);
+      assert getDistributionManager(c3).isLocal(k3);
+      assert getDistributionManager(c4).isLocal(k3);
+
+      BaseDistFunctionalTest.MagicKey k4 = new BaseDistFunctionalTest.MagicKey(c4);
+      assert getDistributionManager(c1).isLocal(k4);
+      assert !getDistributionManager(c2).isLocal(k4);
+      assert !getDistributionManager(c3).isLocal(k4);
+      assert getDistributionManager(c4).isLocal(k4);
+   }
 }

Added: trunk/core/src/test/java/org/infinispan/distribution/RehashJoinTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/distribution/RehashJoinTest.java	                        (rev 0)
+++ trunk/core/src/test/java/org/infinispan/distribution/RehashJoinTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -0,0 +1,175 @@
+package org.infinispan.distribution;
+
+import org.infinispan.Cache;
+import org.infinispan.manager.CacheManager;
+import org.infinispan.remoting.transport.Address;
+import org.infinispan.test.TestingUtil;
+import org.infinispan.util.logging.Log;
+import org.infinispan.util.logging.LogFactory;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+ at Test(testName = "distribution.RehashJoinTest", groups = "functional")
+public class RehashJoinTest extends BaseDistFunctionalTest {
+
+   Log log = LogFactory.getLog(RehashJoinTest.class);
+   CacheManager joinerManager;
+
+   public RehashJoinTest() {
+      cleanup = CleanupPhase.AFTER_METHOD;
+   }
+
+   @Override
+   protected void createCacheManagers() throws Throwable {
+      super.createCacheManagers();
+      joinerManager = addClusterEnabledCacheManager();
+      joinerManager.defineConfiguration(cacheName, configuration);
+   }
+
+   public void testRehashOnJoin() {
+      MagicKey k1 = new MagicKey(c1, "k1");
+      MagicKey k2 = new MagicKey(c2, "k2");
+      MagicKey k3 = new MagicKey(c3, "k3");
+      MagicKey k4 = new MagicKey(c4, "k4");
+
+      List<MagicKey> keys = new ArrayList<MagicKey>(4);
+      keys.add(k1);
+      keys.add(k2);
+      keys.add(k3);
+      keys.add(k4);
+
+      int i = 0;
+      for (Cache<Object, String> c : caches) c.put(keys.get(i++), "v" + i);
+
+      i = 0;
+      for (MagicKey key : keys) assertOnAllCachesAndOwnership(key, "v" + ++i);
+
+      log.info("***>>> Firing up new joiner!");
+
+      // now fire up a new joiner
+      Cache<Object, String> joiner = joinerManager.getCache(cacheName);
+
+      // need to wait for the joiner to, well, join.
+      TestingUtil.blockUntilViewsReceived(60000, cacheManagers.toArray(new CacheManager[cacheManagers.size()]));
+
+      // where does the joiner sit in relation to the other caches?
+      int joinerPos = locateJoiner(joinerManager.getAddress());
+
+      log.info("***>>> Joiner is in position " + joinerPos);
+
+      caches.add(joinerPos, joiner);
+      i = 0;
+      for (MagicKey key : keys) assertOnAllCachesAndOwnership(key, "v" + ++i);
+
+      assertProperConsistentHashOnAllCaches();
+   }
+
+   public void testMultipleJoiners() throws InterruptedException {
+
+      // have all JOIN phases completed?
+      for (Cache c : caches) {
+         DistributionManagerImpl dmi = (DistributionManagerImpl) getDistributionManager(c);
+         assert !dmi.rehashInProgress : "Cache " + addressOf(c) + " still has rehashInProgress=true!";
+         assert dmi.rehashQueue.isEmpty() : "Cache " + addressOf(c) + " still has queued RehashTasks!";
+         assert !(dmi.getConsistentHash() instanceof UnionConsistentHash) : "Cache " + addressOf(c) + " still using a UnionConsistentHash!";
+      }
+
+      MagicKey k1 = new MagicKey(c1, "k1");
+      MagicKey k2 = new MagicKey(c2, "k2");
+      MagicKey k3 = new MagicKey(c3, "k3");
+      MagicKey k4 = new MagicKey(c4, "k4");
+
+      List<MagicKey> keys = new ArrayList<MagicKey>(4);
+      keys.add(k1);
+      keys.add(k2);
+      keys.add(k3);
+      keys.add(k4);
+
+      int i = 0;
+      for (Cache<Object, String> c : caches) c.put(keys.get(i++), "v" + i);
+
+      i = 0;
+      for (MagicKey key : keys) assertOnAllCachesAndOwnership(key, "v" + ++i);
+
+      int numNewJoiners = 3; // in addition to the 1 joiner defined in createCacheManagers()!
+      int numJoiners = numNewJoiners + 1;
+      final CacheManager[] joinerManagers = new CacheManager[numJoiners];
+      final Cache[] joiners = new Cache[numJoiners];
+      joinerManagers[0] = joinerManager;
+
+      for (i = 1; i < numJoiners; i++) {
+         joinerManagers[i] = addClusterEnabledCacheManager();
+         joinerManagers[i].defineConfiguration(cacheName, configuration);
+      }
+
+      log.info("***>>> Firing up {0} new joiners!", numJoiners);
+
+      // now fire up a new joiners, in separate threads.
+      final CountDownLatch joinLatch = new CountDownLatch(1);
+      Thread[] threads = new Thread[numJoiners];
+
+      for (i = 0; i < numJoiners; i++) {
+         final int idx = i;
+         threads[idx] = new Thread() {
+            public void run() {
+               try {
+                  joinLatch.await();
+               } catch (InterruptedException e) {
+                  log.error(e);
+               }
+               joiners[idx] = joinerManagers[idx].getCache(cacheName);
+            }
+         };
+         threads[idx].start();
+      }
+
+      joinLatch.countDown();
+
+      for (Thread t : threads) t.join();
+
+      CacheManager[] cacheManagerArray = cacheManagers.toArray(new CacheManager[cacheManagers.size()]);
+      log.info("Number of cache manager views to wait for: {0}", cacheManagerArray.length);
+
+      // need to wait for the joiner to, well, join.
+      TestingUtil.blockUntilViewsReceived(240000, cacheManagerArray);
+      // where do the joiners sit in relation to the other caches?
+
+      waitForJoinTasksToComplete(120000, joiners);
+
+      // need to wait a *short while* before we attempt to locate joiners, since post-join invalidation messages are sent async.
+      // TODO replace this with some form of command detection on remote nodes.
+      // join tasks happen sequentially as well, so this needs some time to finish
+      TestingUtil.sleepThread(1000);
+
+      int[] joinersPos = new int[numJoiners];
+      for (i = 0; i < numJoiners; i++) joinersPos[i] = locateJoiner(joinerManagers[i].getAddress());
+
+      log.info("***>>> Joiners are in positions " + Arrays.toString(joinersPos));
+      for (i = 0; i < numJoiners; i++) {
+         if (joinersPos[i] > caches.size())
+            caches.add(joiners[i]);
+         else
+            caches.add(joinersPos[i], joiners[i]);
+      }
+      i = 0;
+      for (MagicKey key : keys) assertOnAllCachesAndOwnership(key, "v" + ++i);
+
+      assertProperConsistentHashOnAllCaches();
+   }
+
+   private int locateJoiner(Address joinerAddress) {
+      for (Cache c : Arrays.asList(c1, c2, c3, c4)) {
+         DefaultConsistentHash dch = getDefaultConsistentHash(c, 120000);
+         int i = 0;
+         for (Address a : dch.positions.values()) {
+            if (a.equals(joinerAddress)) return i;
+            i++;
+         }
+      }
+      throw new RuntimeException("Cannot locate joiner! Joiner is [" + joinerAddress + "]");
+   }
+}


Property changes on: trunk/core/src/test/java/org/infinispan/distribution/RehashJoinTest.java
___________________________________________________________________
Name: svn:keywords
   + Id Revision
Name: svn:eol-style
   + LF

Modified: trunk/core/src/test/java/org/infinispan/marshall/MarshallersTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/marshall/MarshallersTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/marshall/MarshallersTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -46,6 +46,7 @@
 import org.infinispan.container.entries.TransientCacheValue;
 import org.infinispan.container.entries.TransientMortalCacheEntry;
 import org.infinispan.container.entries.TransientMortalCacheValue;
+import org.infinispan.distribution.DefaultConsistentHash;
 import org.infinispan.loaders.bucket.Bucket;
 import org.infinispan.marshall.jboss.JBossMarshaller;
 import org.infinispan.remoting.responses.ExtendedResponse;
@@ -67,21 +68,20 @@
 import java.util.*;
 
 /**
- * Test for home grown and JBoss Marshalling based marshallers where data written 
- * and size of payloads are compared. It's disabled by default because JBoss 
- * Marshalling for the moment generates bigger payloads in most cases.  
- * 
+ * Test for home grown and JBoss Marshalling based marshallers where data written and size of payloads are compared.
+ * It's disabled by default because JBoss Marshalling for the moment generates bigger payloads in most cases.
+ *
  * @author Galder Zamarreño
  * @since 4.0
  */
 @Test(groups = "functional", testName = "marshall.MarshallersTest", enabled = true)
 public class MarshallersTest {
-   
+
    private final MarshallerImpl home = new MarshallerImpl();
    private GlobalTransactionFactory gtf = new GlobalTransactionFactory();
    private final JBossMarshaller jboss = new JBossMarshaller();
-   private final Marshaller[] marshallers = new Marshaller[] {home, jboss};
-   
+   private final Marshaller[] marshallers = new Marshaller[]{home, jboss};
+
    @BeforeTest
    public void setUp() {
       home.init(Thread.currentThread().getContextClassLoader(), new RemoteCommandFactory());
@@ -97,12 +97,12 @@
       JGroupsAddress address = new JGroupsAddress(new IpAddress(12345));
       checkEqualityAndSize(address);
    }
-   
+
    public void testGlobalTransactionMarshalling() throws Exception {
       GlobalTransaction gtx = gtf.newGlobalTransaction(new JGroupsAddress(new IpAddress(12345)), false);
       checkEqualityAndSize(gtx);
    }
-   
+
    public void testListMarshalling() throws Exception {
       List l1 = new ArrayList();
       List l2 = new LinkedList();
@@ -114,7 +114,7 @@
       checkEqualityAndSize(l1);
       checkEqualityAndSize(l2);
    }
-   
+
    public void testMapMarshalling() throws Exception {
       Map m1 = new HashMap();
       Map m2 = new TreeMap();
@@ -129,7 +129,7 @@
       Map m5 = Immutables.immutableMapWrap(m3);
       checkEqualityAndSize(m1);
       checkEqualityAndSize(m2);
-      
+
       List<Integer> sizes = new ArrayList<Integer>(2);
       for (Marshaller marshaller : marshallers) {
          byte[] bytes = marshaller.objectToByteBuffer(m4);
@@ -143,7 +143,7 @@
 
       checkEqualityAndSize(m5);
    }
-   
+
    public void testSetMarshalling() throws Exception {
       Set s1 = new HashSet();
       Set s2 = new TreeSet();
@@ -168,12 +168,12 @@
       List l = Collections.singletonList(gtx);
       checkEqualityAndSize(l);
    }
-   
+
    public void testTransactionLogMarshalling() throws Exception {
       GlobalTransaction gtx = gtf.newGlobalTransaction(new JGroupsAddress(new IpAddress(12345)), false);
       PutKeyValueCommand command = new PutKeyValueCommand("k", "v", false, null, 0, 0);
       TransactionLog.LogEntry entry = new TransactionLog.LogEntry(gtx, command);
-      
+
       List<Integer> sizes = new ArrayList<Integer>(2);
       for (Marshaller marshaller : marshallers) {
          byte[] bytes = marshaller.objectToByteBuffer(entry);
@@ -186,7 +186,7 @@
       }
       assert sizes.get(1) < sizes.get(0) : "JBoss Marshaller should write less bytes: bytesJBoss=" + sizes.get(1) + ", bytesHome=" + sizes.get(0);
    }
-   
+
    public void testImmutableResponseMarshalling() throws Exception {
       checkEqualityAndSize(RequestIgnoredResponse.INSTANCE);
       checkEqualityAndSize(UnsuccessfulResponse.INSTANCE);
@@ -208,7 +208,7 @@
       }
       assert sizes.get(1) < sizes.get(0) : "JBoss Marshaller should write less bytes: bytesJBoss=" + sizes.get(1) + ", bytesHome=" + sizes.get(0);
    }
-   
+
    public void testReplicableCommandsMarshalling() throws Exception {
       StateTransferControlCommand c1 = new StateTransferControlCommand(true);
 
@@ -252,7 +252,7 @@
          byte[] bytes = marshaller.objectToByteBuffer(c7);
          InvalidateCommand rc7 = (InvalidateCommand) marshaller.objectFromByteBuffer(bytes);
          assert rc7.getCommandId() == c7.getCommandId() : "Writen[" + c7.getCommandId() + "] and read[" + rc7.getCommandId() + "] objects should be the same";
-         assert Arrays.equals(rc7.getParameters(), c7.getParameters()) : "Writen[" + c7.getParameters() + "] and read[" + rc7.getParameters() + "] objects should be the same";         
+         assert Arrays.equals(rc7.getParameters(), c7.getParameters()) : "Writen[" + c7.getParameters() + "] and read[" + rc7.getParameters() + "] objects should be the same";
          sizes.add(bytes.length);
       }
       assert sizes.get(1) < sizes.get(0) : "JBoss Marshaller should write less bytes: bytesJBoss=" + sizes.get(1) + ", bytesHome=" + sizes.get(0);
@@ -267,7 +267,7 @@
          sizes.add(bytes.length);
       }
       assert sizes.get(1) < sizes.get(0) : "JBoss Marshaller should write less bytes: bytesJBoss=" + sizes.get(1) + ", bytesHome=" + sizes.get(0);
-      
+
       ReplaceCommand c8 = new ReplaceCommand("key", "oldvalue", "newvalue", 0, 0);
       checkEqualityAndSize(c8);
 
@@ -277,7 +277,7 @@
          byte[] bytes = marshaller.objectToByteBuffer(c9);
          ClearCommand rc9 = (ClearCommand) marshaller.objectFromByteBuffer(bytes);
          assert rc9.getCommandId() == c9.getCommandId() : "Writen[" + c9.getCommandId() + "] and read[" + rc9.getCommandId() + "] objects should be the same";
-         assert Arrays.equals(rc9.getParameters(), c9.getParameters()) : "Writen[" + c9.getParameters() + "] and read[" + rc9.getParameters() + "] objects should be the same";         
+         assert Arrays.equals(rc9.getParameters(), c9.getParameters()) : "Writen[" + c9.getParameters() + "] and read[" + rc9.getParameters() + "] objects should be the same";
          sizes.add(bytes.length);
       }
       assert sizes.get(1) < sizes.get(0) : "JBoss Marshaller should write less bytes: bytesJBoss=" + sizes.get(1) + ", bytesHome=" + sizes.get(0);
@@ -318,7 +318,7 @@
       TransientMortalCacheEntry entry4 = (TransientMortalCacheEntry) InternalEntryFactory.create("key", "value", System.currentTimeMillis() - 1000, 200000, System.currentTimeMillis(), 4000000);
       checkEqualityAndSize(entry4);
    }
-   
+
    public void testInternalCacheValueMarshalling() throws Exception {
       byte[] bytes = null;
       ImmortalCacheValue value1 = (ImmortalCacheValue) InternalEntryFactory.createValue("value", System.currentTimeMillis() - 1000, -1, System.currentTimeMillis(), -1);
@@ -340,7 +340,7 @@
          sizes.add(bytes.length);
       }
       assert sizes.get(1) < sizes.get(0) : "JBoss Marshaller should write less bytes: bytesJBoss=" + sizes.get(1) + ", bytesHome=" + sizes.get(0);
-         
+
       sizes.clear();
       for (Marshaller marshaller : marshallers) {
          TransientCacheValue value3 = (TransientCacheValue) InternalEntryFactory.createValue("value", System.currentTimeMillis() - 1000, -1, System.currentTimeMillis(), 4000000);
@@ -350,7 +350,7 @@
          sizes.add(bytes.length);
       }
       assert sizes.get(1) < sizes.get(0) : "JBoss Marshaller should write less bytes: bytesJBoss=" + sizes.get(1) + ", bytesHome=" + sizes.get(0);
-      
+
       sizes.clear();
       for (Marshaller marshaller : marshallers) {
          TransientMortalCacheValue value4 = (TransientMortalCacheValue) InternalEntryFactory.createValue("value", System.currentTimeMillis() - 1000, 200000, System.currentTimeMillis(), 4000000);
@@ -361,7 +361,7 @@
       }
       assert sizes.get(1) < sizes.get(0) : "JBoss Marshaller should write less bytes: bytesJBoss=" + sizes.get(1) + ", bytesHome=" + sizes.get(0);
    }
-   
+
    public void testBucketMarshalling() throws Exception {
       ImmortalCacheEntry entry1 = (ImmortalCacheEntry) InternalEntryFactory.create("key", "value", System.currentTimeMillis() - 1000, -1, System.currentTimeMillis(), -1);
       MortalCacheEntry entry2 = (MortalCacheEntry) InternalEntryFactory.create("key", "value", System.currentTimeMillis() - 1000, 200000, System.currentTimeMillis(), -1);
@@ -373,17 +373,30 @@
       b.addEntry(entry2);
       b.addEntry(entry3);
       b.addEntry(entry4);
-      
+
       List<Integer> sizes = new ArrayList<Integer>(2);
       for (Marshaller marshaller : marshallers) {
          byte[] bytes = marshaller.objectToByteBuffer(b);
          Bucket rb = (Bucket) marshaller.objectFromByteBuffer(bytes);
-         assert rb.getEntries().equals(b.getEntries()) : "Writen[" + b.getEntries() + "] and read[" + rb.getEntries() + "] objects should be the same";         
+         assert rb.getEntries().equals(b.getEntries()) : "Writen[" + b.getEntries() + "] and read[" + rb.getEntries() + "] objects should be the same";
          sizes.add(bytes.length);
       }
       assert sizes.get(1) < sizes.get(0) : "JBoss Marshaller should write less bytes: bytesJBoss=" + sizes.get(1) + ", bytesHome=" + sizes.get(0);
    }
-   
+
+   public void testConsistentHash() throws Exception {
+      DefaultConsistentHash dch = new DefaultConsistentHash();
+      List<Address> caches = new LinkedList<Address>();
+      caches.add(new JGroupsAddress(new org.jgroups.util.UUID()));
+      caches.add(new JGroupsAddress(new org.jgroups.util.UUID()));
+      caches.add(new JGroupsAddress(new org.jgroups.util.UUID()));
+      dch.setCaches(caches);
+
+      byte[] bytes = jboss.objectToByteBuffer(dch);
+      DefaultConsistentHash dch2 = (DefaultConsistentHash) jboss.objectFromByteBuffer(bytes);
+      assert dch2.equals(dch);
+   }
+
    protected void checkEqualityAndSize(Object writeObj) throws Exception {
       int bytesH = marshallAndAssertEquality(home, writeObj);
       int bytesJ = marshallAndAssertEquality(jboss, writeObj);

Modified: trunk/core/src/test/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierImplTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierImplTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierImplTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -28,11 +28,11 @@
    public void testNotifyViewChanged() {
       Address a = EasyMock.createNiceMock(Address.class);
       List<Address> addresses = Collections.emptyList();
-      n.notifyViewChange(addresses, a, 100);
+      n.notifyViewChange(addresses, addresses, a, 100);
 
       assert cl.invocationCount == 1;
       assert ((ViewChangedEvent) cl.getEvent()).getLocalAddress() == a;
-      assert ((ViewChangedEvent) cl.getEvent()).getNewMemberList() == addresses;
+      assert ((ViewChangedEvent) cl.getEvent()).getNewMembers() == addresses;
       assert ((ViewChangedEvent) cl.getEvent()).getViewId() == 100;
       assert cl.getEvent().getType() == Event.Type.VIEW_CHANGED;
    }

Modified: trunk/core/src/test/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/notifications/cachemanagerlistener/CacheManagerNotifierTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -42,7 +42,7 @@
       CacheManagerNotifier mockNotifier = createMock(CacheManagerNotifier.class);
       CacheManagerNotifier origNotifier = TestingUtil.replaceComponent(cm1, CacheManagerNotifier.class, mockNotifier, true);
       try {
-         mockNotifier.notifyViewChange(isA(List.class), eq(myAddress), anyInt());
+         mockNotifier.notifyViewChange(isA(List.class), isA(List.class), eq(myAddress), anyInt());
          replay(mockNotifier);
          // start a second cache.
          Cache c2 = cm2.getCache("cache");

Modified: trunk/core/src/test/java/org/infinispan/tx/ReplDeadlockDetectionTest.java
===================================================================
--- trunk/core/src/test/java/org/infinispan/tx/ReplDeadlockDetectionTest.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/core/src/test/java/org/infinispan/tx/ReplDeadlockDetectionTest.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -25,6 +25,7 @@
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 
@@ -32,7 +33,6 @@
  * Functional test for deadlock detection.
  *
  * @author Mircea.Markus at jboss.com
- *
  */
 @Test(testName = "tx.ReplDeadlockDetectionTest", groups = "functional")
 public class ReplDeadlockDetectionTest extends MultipleCacheManagersTest {
@@ -242,15 +242,15 @@
          this.replicationLatch = replicationLatch;
       }
 
-      public List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue, ResponseFilter responseFilter) {
+      public List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue, ResponseFilter responseFilter) {
          return realOne.invokeRemotely(recipients, rpcCommand, mode, timeout, usePriorityQueue, responseFilter);
       }
 
-      public List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue) {
+      public List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout, boolean usePriorityQueue) {
          return realOne.invokeRemotely(recipients, rpcCommand, mode, timeout, usePriorityQueue);
       }
 
-      public List<Response> invokeRemotely(List<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout) throws Exception {
+      public List<Response> invokeRemotely(Collection<Address> recipients, ReplicableCommand rpcCommand, ResponseMode mode, long timeout) throws Exception {
          return realOne.invokeRemotely(recipients, rpcCommand, mode, timeout);
       }
 
@@ -285,23 +285,23 @@
          realOne.broadcastRpcCommandInFuture(rpc, usePriorityQueue, future);
       }
 
-      public void invokeRemotely(List<Address> recipients, ReplicableCommand rpc, boolean sync) throws ReplicationException {
+      public void invokeRemotely(Collection<Address> recipients, ReplicableCommand rpc, boolean sync) throws ReplicationException {
          realOne.invokeRemotely(recipients, rpc, sync);
       }
 
-      public void invokeRemotely(List<Address> recipients, ReplicableCommand rpc, boolean sync, boolean usePriorityQueue) throws ReplicationException {
+      public void invokeRemotely(Collection<Address> recipients, ReplicableCommand rpc, boolean sync, boolean usePriorityQueue) throws ReplicationException {
          realOne.invokeRemotely(recipients, rpc, sync, usePriorityQueue);
       }
 
-      public void invokeRemotelyInFuture(List<Address> recipients, ReplicableCommand rpc, NotifyingNotifiableFuture<Object> future) {
+      public void invokeRemotelyInFuture(Collection<Address> recipients, ReplicableCommand rpc, NotifyingNotifiableFuture<Object> future) {
          realOne.invokeRemotelyInFuture(recipients, rpc, future);
       }
 
-      public void invokeRemotelyInFuture(List<Address> recipients, ReplicableCommand rpc, boolean usePriorityQueue, NotifyingNotifiableFuture<Object> future) {
+      public void invokeRemotelyInFuture(Collection<Address> recipients, ReplicableCommand rpc, boolean usePriorityQueue, NotifyingNotifiableFuture<Object> future) {
          realOne.invokeRemotelyInFuture(recipients, rpc, usePriorityQueue, future);
       }
 
-      public void invokeRemotelyInFuture(List<Address> recipients, ReplicableCommand rpc, boolean usePriorityQueue, NotifyingNotifiableFuture<Object> future, long timeout) {
+      public void invokeRemotelyInFuture(Collection<Address> recipients, ReplicableCommand rpc, boolean usePriorityQueue, NotifyingNotifiableFuture<Object> future, long timeout) {
          realOne.invokeRemotelyInFuture(recipients, rpc, usePriorityQueue, future, timeout);
       }
 

Modified: trunk/gui-demo/src/main/java/org/infinispan/demo/InfinispanDemo.java
===================================================================
--- trunk/gui-demo/src/main/java/org/infinispan/demo/InfinispanDemo.java	2009-08-13 14:10:18 UTC (rev 672)
+++ trunk/gui-demo/src/main/java/org/infinispan/demo/InfinispanDemo.java	2009-08-13 15:16:24 UTC (rev 673)
@@ -441,7 +441,7 @@
    public class CacheListener {
       @ViewChanged
       public void viewChangeEvent(ViewChangedEvent e) {
-         updateClusterTable(e.getNewMemberList());
+         updateClusterTable(e.getNewMembers());
       }
 
       @CacheEntryModified



More information about the infinispan-commits mailing list