[jbosscache-dev] Re: MUX issues with JGroups 2.6.Beta1 and JBoss Cache

I don't think this is Hudson related - it hangs locally too, waiting for state. E.g., thread dump below is from o.j.c.multiplexer.SyncReplTxTest#testPut() Oct 4, 2007 5:52:49 PM org.jgroups.JChannel init INFO: JGroups version: 2.6.0 beta-1 ------------------------------------------------------- GMS: address is 127.0.0.1:49380 ------------------------------------------------------- Oct 4, 2007 5:52:51 PM org.jboss.cache.CacheImpl$MembershipListenerAdaptor viewAccepted INFO: viewAccepted(): [127.0.0.1:49380|0] [127.0.0.1:49380] Full thread dump Java HotSpot(TM) Client VM (1.5.0_07-87 mixed mode, sharing): "Multiplexer-2" daemon prio=5 tid=0x00518e10 nid=0x18d5c00 waiting on condition [0xb1193000..0xb1193d10] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:118) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:681) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:736) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1064) at java.util.concurrent.SynchronousQueue$Node.waitForPut(SynchronousQueue.java:265) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:400) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) at java.lang.Thread.run(Thread.java:613) "Timer,jbc-test,127.0.0.1:49380" daemon prio=5 tid=0x00519360 nid=0x18d5800 waiting on condition [0xb1112000..0xb1112d10] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803) at java.util.concurrent.DelayQueue.take(DelayQueue.java:135) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) at java.lang.Thread.run(Thread.java:613) "OOB Thread,jbc-test,127.0.0.1:49380" prio=10 tid=0x00519830 nid=0x18d5400 waiting on condition [0xb1091000..0xb1091d10] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:118) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:681) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:736) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1064) at java.util.concurrent.SynchronousQueue$Node.waitForPut(SynchronousQueue.java:265) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:400) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) at java.lang.Thread.run(Thread.java:613) "Timer,jbc-test,127.0.0.1:49380" daemon prio=5 tid=0x0051b710 nid=0x18d5000 waiting on condition [0xb1010000..0xb1010d10] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803) at java.util.concurrent.DelayQueue.take(DelayQueue.java:135) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) at java.lang.Thread.run(Thread.java:613) "FD_SOCK ServerSocket acceptor thread (channel=jbc-test)" daemon prio=5 tid=0x00517970 nid=0x18d4800 runnable [0xb0f8f000..0xb0f8fd10] at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:384) - locked <0x265a1380> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:450) at java.net.ServerSocket.accept(ServerSocket.java:421) at org.jgroups.protocols.FD_SOCK$ServerSocketHandler.run(FD_SOCK.java:1054) at java.lang.Thread.run(Thread.java:613) "UDP mcast receiver,jbc-test,127.0.0.1:49380" prio=10 tid=0x005188c0 nid=0x18d4400 runnable [0xb0f0e000..0xb0f0ed10] at java.net.PlainDatagramSocketImpl.receive0(Native Method) - locked <0x2660a338> (a java.net.PlainDatagramSocketImpl) at java.net.PlainDatagramSocketImpl.receive(PlainDatagramSocketImpl.java:136) - locked <0x2660a338> (a java.net.PlainDatagramSocketImpl) at java.net.DatagramSocket.receive(DatagramSocket.java:712) - locked <0x265a0040> (a java.net.DatagramPacket) - locked <0x2660a0e8> (a java.net.MulticastSocket) at org.jgroups.protocols.UDP.run(UDP.java:247) at java.lang.Thread.run(Thread.java:613) "UDP ucast receiver,jbc-test,127.0.0.1:49380" prio=5 tid=0x00516c30 nid=0x18d6c00 runnable [0xb0e8d000..0xb0e8dd10] at java.net.PlainDatagramSocketImpl.receive0(Native Method) - locked <0x2660a2f0> (a java.net.PlainDatagramSocketImpl) at java.net.PlainDatagramSocketImpl.receive(PlainDatagramSocketImpl.java:136) - locked <0x2660a2f0> (a java.net.PlainDatagramSocketImpl) at java.net.DatagramSocket.receive(DatagramSocket.java:712) - locked <0x26590010> (a java.net.DatagramPacket) - locked <0x2660a0c8> (a java.net.DatagramSocket) at org.jgroups.protocols.UDP$UcastReceiver.run(UDP.java:914) at java.lang.Thread.run(Thread.java:613) "DiagnosticsHandler,jbc-test,127.0.0.1:49380" daemon prio=5 tid=0x00516340 nid=0x18d4c00 runnable [0xb0e0c000..0xb0e0cd10] at java.net.PlainDatagramSocketImpl.receive0(Native Method) - locked <0x26600730> (a java.net.PlainDatagramSocketImpl) at java.net.PlainDatagramSocketImpl.receive(PlainDatagramSocketImpl.java:136) - locked <0x26600730> (a java.net.PlainDatagramSocketImpl) at java.net.DatagramSocket.receive(DatagramSocket.java:712) - locked <0x26600770> (a java.net.DatagramPacket) - locked <0x26600790> (a java.net.MulticastSocket) at org.jgroups.protocols.TP$DiagnosticsHandler.run(TP.java:1813) at java.lang.Thread.run(Thread.java:613) "Timer,jbc-test,127.0.0.1:49380" daemon prio=5 tid=0x0051cde0 nid=0x185be00 waiting on condition [0xb0d0a000..0xb0d0ad10] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:146) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803) at java.util.concurrent.DelayQueue.take(DelayQueue.java:135) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:504) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:497) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:674) at java.lang.Thread.run(Thread.java:613) "ReaderThread" prio=5 tid=0x0050d210 nid=0x1845a00 runnable [0xb0c89000..0xb0c89d10] at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.read(SocketInputStream.java:129) at sun.nio.cs.StreamDecoder$CharsetSD.readBytes(StreamDecoder.java:411) at sun.nio.cs.StreamDecoder$CharsetSD.implRead(StreamDecoder.java:453) at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:183) - locked <0x26ac48a8> (a java.io.InputStreamReader) at java.io.InputStreamReader.read(InputStreamReader.java:167) at java.io.BufferedReader.fill(BufferedReader.java:136) at java.io.BufferedReader.readLine(BufferedReader.java:299) - locked <0x26ac48a8> (a java.io.InputStreamReader) at java.io.BufferedReader.readLine(BufferedReader.java:362) at org.testng.remote.strprotocol.StringMessageSenderHelper$ReaderThread.run(StringMessageSenderHelper.java:190) "Low Memory Detector" daemon prio=5 tid=0x00508fe0 nid=0x1817000 runnable [0x00000000..0x00000000] "CompilerThread0" daemon prio=9 tid=0x005085e0 nid=0x1816c00 waiting on condition [0x00000000..0xb0b0674c] "Signal Dispatcher" daemon prio=9 tid=0x005080d0 nid=0x1816800 waiting on condition [0x00000000..0x00000000] "Finalizer" daemon prio=8 tid=0x00507820 nid=0x1812400 in Object.wait() [0xb0a04000..0xb0a04d10] at java.lang.Object.wait(Native Method) - waiting on <0x26a84ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116) - locked <0x26a84ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00507420 nid=0x1811600 in Object.wait() [0xb0983000..0xb0983d10] at java.lang.Object.wait(Native Method) - waiting on <0x26a84b48> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:474) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x26a84b48> (a java.lang.ref.Reference$Lock) "main" prio=5 tid=0x005014f0 nid=0x1805600 in Object.wait() [0xb07ff000..0xb08000dc] at java.lang.Object.wait(Native Method) - waiting on <0x26bca008> (a java.lang.Object) at java.lang.Object.wait(Object.java:474) at org.jboss.cache.CacheImpl$MessageListenerAdaptor.waitForState(CacheImpl.java:3350) - locked <0x26bca008> (a java.lang.Object) at org.jboss.cache.CacheImpl.internalStart(CacheImpl.java:761) at org.jboss.cache.CacheImpl.start(CacheImpl.java:710) at org.jboss.cache.replicated.SyncReplTxTest.initCaches(SyncReplTxTest.java:105) at org.jboss.cache.replicated.SyncReplTxTest.testPut(SyncReplTxTest.java:893) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:585) at org.testng.internal.MethodHelper.invokeMethod(MethodHelper.java:604) at org.testng.internal.Invoker.invokeMethod(Invoker.java:470) at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:564) at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:830) at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:125) at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:109) at org.testng.TestRunner.runWorkers(TestRunner.java:678) at org.testng.TestRunner.privateRun(TestRunner.java:624) at org.testng.TestRunner.run(TestRunner.java:495) at org.testng.SuiteRunner.runTest(SuiteRunner.java:300) at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:295) at org.testng.SuiteRunner.privateRun(SuiteRunner.java:275) at org.testng.SuiteRunner.run(SuiteRunner.java:190) at org.testng.TestNG.createAndRunSuiteRunners(TestNG.java:792) at org.testng.TestNG.runSuitesLocally(TestNG.java:765) at org.testng.TestNG.run(TestNG.java:699) at org.testng.remote.RemoteTestNG.run(RemoteTestNG.java:73) at org.testng.remote.RemoteTestNG.main(RemoteTestNG.java:122) "VM Thread" prio=9 tid=0x00506c10 nid=0x1811200 runnable "VM Periodic Task Thread" prio=9 tid=0x0050a7f0 nid=0x1817400 waiting on condition "Exception Catcher Thread" prio=10 tid=0x00501710 nid=0x1806600 runnable On 4 Oct 2007, at 17:49, Bela Ban wrote: > Is this Hudson related, or did the *same* tests run fine with > CruiseControl ? > > Manik Surtani wrote: >> Guys, >> >> As you may know I finally have Hudson behaving reasonably well, >> running the JBC test suite off trunk. To do this I've had to >> disable all MUX related tests since these caused the test suite to >> deadlock. I haven't delved further as to why, but will be doing so >> today. I haven't tried rolling back to JG 2.5 as a control test >> (since code changes have now been introduced, adding a few >> dependencies on JG 2.6's API), but again stuff I will be >> investigating later. >> >> Just an early heads-up. >> >> Cheers, >> -- >> Manik Surtani >> >> Lead, JBoss Cache >> JBoss, a division of Red Hat >> >> >> > > -- > Bela Ban > Lead JGroups / Clustering Team > JBoss - a division of Red Hat > -- Manik Surtani Lead, JBoss Cache JBoss, a division of Red Hat