Hi,

I've looked into the test suite failures on hudson and they are mainly caused by the fact that cluster is not formed.
I've switched back from TCP+MPING to TCP+PING and that solved the problem (there are only 3 errors in core module now, still have to look into that).
In order to narrow down the issue I've wrote a test that created two clusters, each cluster having 3 channels (entire test is at jgroups level, no ISPN involved)  
The clusters use two different MPING protocols: different address and ports:  http://fisheye.jboss.org/browse/Infinispan/trunk/core/src/test/java/org/infinispan/test/fwk/TcpMPingEnvironmentTest.java?r=640
It passes on my mac, win and even on the ATL cluster when not started by hudson process (!!!), but fails when run by hudson[1].

Can you please take a look ? 


[1] Here are the System.getProperties dumped from the hudson jvm.
System props are {JOB_NAME=Infinispan-trunk-JDK5-tcp, java.vendor=Sun Microsystems Inc., PWD=/qa/services/hudson, localRepository=/home/hudson/.m2/repository, sun.java.launcher=SUN_STANDARD, sun.management.compiler=HotSpot Server Compiler, PATH=/qa/tools/opt/apache-ant-1.6.5/bin:/qa/tools/opt/maven-2.0.9/bin:/qa/tools/opt/jdk1.5.0_15/bin:/qa/tools/opt/subversion-1.4.0/bin:/usr/xpg4/bin:/usr/xpg4/bin:/usr/xpg4/bin:/usr/kerberos/bin:/usr/xpg4/bin:/usr/local/bin:/bin:/usr/bin:/opt/maven/bin:/opt/subversion/bin:/usr/X11R6/bin:/opt/maven/bin:/opt/subversion/bin:/home/hudson/bin, os.name=Linux, JAVA_FOR_SLAVE=/qa/tools/opt/jdk1.6.0_11, sun.boot.class.path=/qa/tools/opt/jdk1.5.0_15/jre/lib/rt.jar:/qa/tools/opt/jdk1.5.0_15/jre/lib/i18n.jar:/qa/tools/opt/jdk1.5.0_15/jre/lib/sunrsasign.jar:/qa/tools/opt/jdk1.5.0_15/jre/lib/jsse.jar:/qa/tools/opt/jdk1.5.0_15/jre/lib/jce.jar:/qa/tools/opt/jdk1.5.0_15/jre/lib/charsets.jar:/qa/tools/opt/jdk1.5.0_15/jre/classes, ANT_OPTS=-Xmx700m , EXECUTOR_NUMBER=0, java.vm.specification.vendor=Sun Microsystems Inc., java.runtime.version=1.5.0_15-b04, HUDSON_STATIC_ENV=/home/hudson/static_build_env, ANT_HOME=/qa/tools/opt/apache-ant-1.6.5, LD_LIBRARY_PATH=/qa/tools/opt/jdk1.6.0_11/jre/lib/i386/server:/qa/tools/opt/jdk1.6.0_11/jre/lib/i386:/qa/tools/opt/jdk1.6.0_11/jre/../lib/i386:/qa/tools/opt/subversion-1.4.0/lib/:, user.name=hudson, SSH_CONNECTION=::ffff:10.16.88.204 47991 ::ffff:10.16.88.186 22, protocol.stack=tcp, SSH_CLIENT=::ffff:10.16.88.204 47991 22, SOURCE_REPO=/qa/tools/src, MYTESTIPS=10.16.93.249 10.16.93.250, MYTESTIP_2=10.16.93.250, MYTESTIP_1=10.16.93.249, java.net.preferIpv4Stack=true, JAVA16=jdk1.6.0_11, user.language=en, JAVA15=jdk1.5.0_15, maven.interceptor.override=/home/hudson/hudson_workspace/maven2.1-interceptor.jar, KDEDIR=/usr, sun.boot.library.path=/qa/tools/opt/jdk1.5.0_15/jre/lib/i386, java.version=1.5.0_15, java.net.preferIPv4Stack=true, user.timezone=America/New_York, sun.arch.data.model=32, NODE_NAME=perf08, NLSPATH=/usr/dt/lib/nls/msg/%L/%N.cat, java.endorsed.dirs=/qa/tools/opt/jdk1.5.0_15/jre/lib/endorsed, BUILD_ID=2009-08-06_16-58-59, sun.cpu.isalist=, sun.jnu.encoding=UTF-8, file.encoding.pkg=sun.io, SHELL=/bin/bash, file.separator=/, java.specification.name=Java Platform API Specification, java.class.version=49.0, user.country=US, java.home=/qa/tools/opt/jdk1.5.0_15/jre, java.vm.info=mixed mode, bind.address=127.0.0.1, LESSOPEN=|/usr/bin/lesspipe.sh %s, os.version=2.6.9-42.0.10.ELsmp, path.separator=:, java.vm.version=1.5.0_15-b04, SHLVL=1, java.awt.printerjob=sun.print.PSPrinterJob, JAVA_HOME=/qa/tools/opt/jdk1.5.0_15, sun.io.unicode.encoding=UnicodeLittle, LOGNAME=hudson, HOME=/home/hudson, maven.interceptor=/home/hudson/hudson_workspace/maven-interceptor.jar, G_BROKEN_FILENAMES=1, user.home=/home/hudson, BUILD_TAG=hudson-Infinispan-trunk-JDK5-tcp-584, HUDSON_URL=http://hudson.qa.jboss.com/hudson/, java.specification.vendor=Sun Microsystems Inc., CLASSPATH=, java.library.path=/qa/tools/opt/jdk1.5.0_15/jre/lib/i386/server:/qa/tools/opt/jdk1.5.0_15/jre/lib/i386:/qa/tools/opt/jdk1.5.0_15/jre/../lib/i386:/qa/tools/opt/jdk1.6.0_11/jre/lib/i386/server:/qa/tools/opt/jdk1.6.0_11/jre/lib/i386:/qa/tools/opt/jdk1.6.0_11/jre/../lib/i386:/qa/tools/opt/subversion-1.4.0/lib/:, java.vendor.url=http://java.sun.com/, NATIVE_TOOLS=/qa/tools/opt, java.vm.vendor=Sun Microsystems Inc., HISTSIZE=1000, maven.home=/qa/tools/opt/maven-2.1.0, java.runtime.name=Java(TM) 2 Runtime Environment, Standard Edition, java.class.path=/home/hudson/hudson_workspace/maven-agent.jar:/qa/tools/opt/maven-2.1.0/boot/classworlds-1.1.jar, MAX_JAVA_MEM=-Xmx700m, java.vm.specification.name=Java Virtual Machine Specification, PATH+JDK=/qa/tools/opt/jdk1.5.0_15/bin, java.vm.specification.version=1.0, MAIL=/var/spool/mail/hudson, M2_HOME=/qa/tools/opt/maven-2.0.9, sun.cpu.endian=little, sun.os.patch.level=unknown, surefire.test.class.path=/home/hudson/hudson_workspace/workspace/Infinispan-trunk-JDK5-tcp/trunk/core/target/test-classes:/home/hudson/hudson_workspace/workspace/Infinispan-trunk-JDK5-tcp/trunk/core/target/classes:/home/hudson/.m2/repository/jgroups/jgroups/2.8.0.Beta2/jgroups-2.8.0.Beta2.jar:/home/hudson/.m2/repository/org/jboss/javaee/jboss-transaction-api/1.0.1.GA/jboss-transaction-api-1.0.1.GA.jar:/home/hudson/.m2/repository/org/jboss/jboss-common-core/2.2.14.GA/jboss-common-core-2.2.14.GA.jar:/home/hudson/.m2/repository/org/jboss/logging/jboss-logging-spi/2.1.0.GA/jboss-logging-spi-2.1.0.GA.jar:/home/hudson/.m2/repository/org/jboss/marshalling/marshalling-api/1.2.0.CR2/marshalling-api-1.2.0.CR2.jar:/home/hudson/.m2/repository/org/jboss/marshalling/river/1.2.0.CR2/river-1.2.0.CR2.jar:/home/hudson/.m2/repository/log4j/log4j/1.2.14/log4j-1.2.14.jar:/home/hudson/.m2/repository/com/thoughtworks/xstream/xstream/1.2/xstream-1.2.jar:/home/hudson/.m2/repository/xpp3/xpp3_min/1.1.3.4.O/xpp3_min-1.1.3.4.O.jar:/home/hudson/.m2/repository/net/jcip/jcip-annotations/1.0/jcip-annotations-1.0.jar:/home/hudson/.m2/repository/org/testng/testng/5.8/testng-5.8-jdk15.jar:/home/hudson/.m2/repository/junit/junit/3.8.1/junit-3.8.1.jar:/home/hudson/.m2/repository/org/easymock/easymockclassextension/2.4/easymockclassextension-2.4.jar:/home/hudson/.m2/repository/org/easymock/easymock/2.4/easymock-2.4.jar:/home/hudson/.m2/repository/cglib/cglib-nodep/2.1_3/cglib-nodep-2.1_3.jar:/home/hudson/.m2/repository/jboss/jbossts/jbossjta/4.4.0.GA/jbossjta-4.4.0.GA.jar:/home/hudson/.m2/repository/javax/xml/bind/jaxb-api/2.1/jaxb-api-2.1.jar:/home/hudson/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/home/hudson/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/home/hudson/.m2/repository/com/sun/xml/bind/jaxb-impl/2.1.3/jaxb-impl-2.1.3.jar:, HOSTNAME=perf08, java.io.tmpdir=/tmp, HUDSON_HOME=/home/hudson/hudson_workspace, MCAST_PORT=45874, LS_COLORS=, java.vendor.url.bug=http://java.sun.com/cgi-bin/bugreport.cgi, SEP=/, MAVEN_OPTS=-Xmx1G, os.arch=i386, java.awt.graphicsenv=sun.awt.X11GraphicsEnvironment, java.ext.dirs=/qa/tools/opt/jdk1.5.0_15/jre/lib/ext, COMMON_TOOLS=/qa/tools/opt, SVN_REVISION=640, user.dir=/home/hudson/hudson_workspace/workspace/Infinispan-trunk-JDK5-tcp/trunk/core, line.separator=
, BUILD_NUMBER=584, java.vm.name=Java HotSpot(TM) Server VM, PSEP=:, basedir=/home/hudson/hudson_workspace/workspace/Infinispan-trunk-JDK5-tcp/trunk/core, LANG=en_US.UTF-8, file.encoding=UTF-8, HUDSON_CONFIG_DIR=/home/hudson/config_repository, infinispan.marshaller.class=${marshaller.class}, WORKSPACE=/home/hudson/hudson_workspace/workspace/Infinispan-trunk-JDK5-tcp, java.specification.version=1.5, INPUTRC=/etc/inputrc, USER=hudson, MCAST_ADDR=227.43.88.186, XFILESEARCHPATH=/usr/dt/app-defaults/%L/Dt}