Invocations fail when the pool exhausts and under heavy load ------------------------------------------------------------ Key: JBREM-409 URL: http://jira.jboss.com/jira/browse/JBREM-409 Project: JBoss Remoting Issue Type: Bug Security Level: Public(Everyone can see) Components: general Affects Versions: 1.4.1 final Reporter: BulentErdemir Assigned To: Tom Elrod Priority: Critical Fix For: 2.0.0.Beta2 (Boon) SocketServerInvoker, when there's no spare thread available in the pool, waits on the clientpool. And when the clientpool is notified, fetches a ServerThread from the ThreadPool when its size() > 0 and notifies that ServerThread by calling its wakeup() method. ServerThread's, on the other hand, after finishing their work, notify the clientpool, add themselves to the threadpool, then notify clientpool. After this notification, ServerThread's start to wait on themselves in the hope to be reused. The bug occurs in the right time frame when SocketServerInvoker notifies the ServerThread but before when the ServerThread starts waiting on itself. So the notification is lost, therefore ServerThread keeps on waiting forever. In other words, ServerThread finishes work, notifies SocketServerInvoker, SocketServerInvoker wakes up, picks the newly available ServerThread, and wakes it up by notifying it (or at least it thinks it wakes the ServerThread up), before the ServerThread gets the chance to wait on itself. Please try the following code. I'd like to hear the comments before supplying a patch. Regards, Bulent Erdemir Code: package org.falez.sandbox.remoting; import org.apache.log4j.BasicConfigurator; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.jboss.remoting.Client; import org.jboss.remoting.InvokerLocator; import org.jboss.remoting.transport.Connector; public class RemotingTest { static { BasicConfigurator.configure(); Logger.getRootLogger().setLevel(Level.INFO); } private Logger logger = Logger.getRootLogger(); public static void main(String[] args) throws Throwable { RemotingTest rt = new RemotingTest(); rt.startServer(); rt.runMultipleClients(Integer.parseInt(args[1])); System.in.read(); } public void startServer() throws Exception { String locatorURI = "socket://localhost:54000/?maxPoolSize=2&timeout=10000"; InvokerLocator locator = new InvokerLocator(locatorURI); Connector connector = new Connector(); connector.setInvokerLocator(locator.getLocatorURI()); connector.create(); SampleInvocationHandler invocationHandler = new SampleInvocationHandler(); connector.start(); connector.addInvocationHandler("sample", invocationHandler); } public void runClient(String clientId) throws Throwable { String locatorURI = "socket://localhost:54000/"; InvokerLocator locator = new InvokerLocator(locatorURI); Client client = new Client(locator); String req =clientId; Object resp = client.invoke(req); System.in.read(); } public void runMultipleClients(int cnt) throws Throwable { for (int i = 0; i < cnt; i++) { Thread t = new Thread(new Runnable() { public void run() { try { Thread.sleep(1000); runClient(Thread.currentThread().getName()); } catch (Throwable e) { logger.error(e); } } }, Integer.toString(i)); t.start(); } } } package org.falez.sandbox.remoting; import javax.management.MBeanServer; import org.apache.log4j.Logger; import org.jboss.remoting.InvocationRequest; import org.jboss.remoting.ServerInvocationHandler; import org.jboss.remoting.ServerInvoker; import org.jboss.remoting.callback.InvokerCallbackHandler; public class SampleInvocationHandler implements ServerInvocationHandler { private static Logger logger = Logger.getRootLogger(); public Object invoke(InvocationRequest invocation) throws Throwable { String parm = (String) invocation.getParameter(); Thread.sleep(5000); System.out.println(Thread.currentThread() + "Invoked " + parm); String s = "response" + parm; return s; } public void setMBeanServer(MBeanServer server) { } public void setInvoker(ServerInvoker invoker) { } public void addListener(InvokerCallbackHandler callbackHandler) { } public void removeListener(InvokerCallbackHandler callbackHandler) { } }