FD_SOCK thread "leak" --------------------- Key: JGRP-246 URL: http://jira.jboss.com/jira/browse/JGRP-246 Project: JGroups Issue Type: Bug Affects Versions: 2.2.8, 2.2.9, 2.3, 2.2.9.1, 2.2.9.2 Environment: SLES 9 Reporter: Bruce Schuchardt Assigned To: Bela Ban Priority: Minor Fix For: 2.4 While investigating a startup problem in an application that has one long-running group member and four other VMs that frequently close their channels and create new ones, I found that the long-running member was piling up a lot of FD_SOCK ClientConnectionHandler threads. When a channel is closed, FD_SOCK closes the socket that ClientConnectionHandler's socket is connected to, but on some operating systems (SLES 9 in this case) this doesn't always cause a socket.read() to exit. All of the ClientConnectionHandler sockets were stuck in this state. I added a write(NORMAL_TERMINATION) before closing the socket, and changed ClientConnectionHandler to look for NORMAL_TERMINATION, and that seems to correct the problem. void stopPingerThread() { if(pinger_thread != null && pinger_thread.isAlive()) { regular_sock_close=true; sendPingTermination(); // PATCH teardownPingSocket(); } pinger_thread=null; } // PATCH: send something so the connection handler can exit synchronized void sendPingTermination() { if (ping_sock != null) { try { ping_sock.getOutputStream().write(NORMAL_TERMINATION); ping_sock.getOutputStream().flush(); } catch (java.io.IOException ie) { if (trace) log.trace("FD_SOCK io exception sending ping termination", ie); } } } void interruptPingerThread() { if(pinger_thread != null && pinger_thread.isAlive()) { regular_sock_close=true; sendPingTermination(); // PATCH teardownPingSocket(); // will wake up the pinger thread. less elegant than Thread.interrupt(), but does the job } } ClientConnectionHandler public void run() { try { synchronized(mutex) { if(client_sock == null) return; in=client_sock.getInputStream(); } // PATCH STARTS HERE int b = 0; do { b = in.read(); } while ( b != ABNORMAL_TERMINATION && b != NORMAL_TERMINATION); // END OF PATCH } catch(IOException io_ex1) { } finally { Socket sock = client_sock; // PATCH: avoid race condition causing NPE if (sock != null && !sock.isClosed()) closeClientSocket(); synchronized(clients) { clients.remove(this); } } } }