[jboss-user] [Remoting] - Re: Unexplained ConnectionValidator Timeout

thammoud do-not-reply at jboss.com
Mon Mar 2 06:52:59 EST 2009


I think that you are seeing the same problem that is described here:

http://www.jboss.org/index.html?module=bb&op=viewtopic&t=151397

I am attaching a version of ConnectionValidator that fixed the problem for us:


  | /*
  | * JBoss, Home of Professional Open Source
  | * Copyright 2005, JBoss Inc., and individual contributors as indicated
  | * by the @authors tag. See the copyright.txt in the distribution for a
  | * full listing of individual contributors.
  | *
  | * This is free software; you can redistribute it and/or modify it
  | * under the terms of the GNU Lesser General Public License as
  | * published by the Free Software Foundation; either version 2.1 of
  | * the License, or (at your option) any later version.
  | *
  | * This software is distributed in the hope that it will be useful,
  | * but WITHOUT ANY WARRANTY; without even the implied warranty of
  | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  | * Lesser General Public License for more details.
  | *
  | * You should have received a copy of the GNU Lesser General Public
  | * License along with this software; if not, write to the Free
  | * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  | * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
  | */
  | 
  | package org.jboss.remoting;
  | 
  | import java.util.ArrayList;
  | import java.util.HashMap;
  | import java.util.List;
  | import java.util.ListIterator;
  | import java.util.Map;
  | import java.util.Timer;
  | import java.util.TimerTask;
  | 
  | import org.jboss.logging.Logger;
  | import org.jboss.remoting.transport.ClientInvoker;
  | import org.jboss.remoting.util.StoppableTimerTask;
  | import org.jboss.remoting.util.TimerUtil;
  | 
  | /**
  |  * @author <a href="mailto:tom.elrod at jboss.com">Tom Elrod</a>
  |  * @author <a href="mailto:ovidiu at jboss.org">Ovidiu Feodorov</a>
  |  * @author <a href="mailto:tlee at redhat.com">Trustin Lee</a>
  |  */
  | public class ConnectionValidator extends TimerTask implements StoppableTimerTask
  | {
  |    // Constants ------------------------------------------------------------------------------------
  | 
  |    private static final Logger log = Logger.getLogger(ConnectionValidator.class.getName());
  | 
  |    /** Configuration map key for ping period. */
  |    public static final String VALIDATOR_PING_PERIOD = "validatorPingPeriod";
  |    
  |    /** Default ping period. Value is 2 seconds. */
  |    public static final long DEFAULT_PING_PERIOD = 2000;
  |    
  |    /** Configuration map key for ping timeout. */
  |    public static final String VALIDATOR_PING_TIMEOUT = "validatorPingTimeout";
  |    
  |    /** Default ping timeout period.  Value is 1 second. */
  |    public static final String DEFAULT_PING_TIMEOUT = "1000";
  |    
  |    /**
  |     * Default number of ping retries.  Value is 1.
  |     * Currently implemented only on socket transport family.
  |     */
  |    public static final String DEFAULT_NUMBER_OF_PING_RETRIES = "1";
  | 
  |    /**
  |     * Key to determine if ConnectionValidator should tie failure to presence
  |     * of active lease on server side.  Default value is "true".
  |     */
  |    public static final String TIE_TO_LEASE = "tieToLease";
  |    /**
  |     * Key to determine whether to stop ConnectionValidator when PING fails.
  |     * Default value is "true".
  |     */
  |    public static final String STOP_LEASE_ON_FAILURE = "stopLeaseOnFailure";
  |    
  |    // Static ---------------------------------------------------------------------------------------
  | 
  |    private static boolean trace = log.isTraceEnabled();
  | 
  |    /**
  |     * Will make $PING$ invocation on server. If sucessful, will return true. Otherwise, will throw
  |     * an exception.
  |     *
  |     * @param locator - locator for the server to ping
  |     * @param config  - any configuration needed for server
  |     * @return true if alive, false if not
  |     */
  |    public static boolean checkConnection(InvokerLocator locator, Map config) throws Throwable
  |    {
  |       boolean pingWorked = false;
  |       Map configMap = createPingConfig(config, null);
  |       int pingTimeout = Integer.parseInt((String) configMap.get(ServerInvoker.TIMEOUT));
  |       ClientInvoker innerClientInvoker = null;
  | 
  |       try
  |       {
  |          innerClientInvoker = InvokerRegistry.createClientInvoker(locator, configMap);
  | 
  |          if (!innerClientInvoker.isConnected())
  |          {
  |             if (trace) { log.trace("inner client invoker not connected, connecting ..."); }
  |             innerClientInvoker.connect();
  |          }
  | 
  |          pingWorked = doCheckConnection(innerClientInvoker, pingTimeout);
  |       }
  |       catch (Throwable throwable)
  |       {
  |          log.debug("ConnectionValidator to connect to server " +
  |             innerClientInvoker.getLocator().getProtocol() + "://" +
  |             innerClientInvoker.getLocator().getHost() + ":" +
  |             innerClientInvoker.getLocator().getPort(), throwable);
  |       }
  |       finally
  |       {
  |          if (innerClientInvoker != null)
  |          {
  |             InvokerRegistry.destroyClientInvoker(locator, configMap);
  |          }
  |       }
  | 
  |       return pingWorked;
  |    }
  | 
  |    private static boolean doCheckConnection(ClientInvoker clientInvoker, int pingTimeout) throws Throwable
  |    {
  |       boolean pingWorked = false;
  | 
  |       try
  |       {
  |          // Sending null client id as don't want to trigger lease on server side. This also means
  |          // that client connection validator will NOT impact client lease, so can not depend on it
  |          // to maintain client lease with the server.
  |          InvocationRequest ir;
  |          ir = new InvocationRequest(null, Subsystem.SELF, "$PING$", null, null, null);
  |          ConnectionCheckThread t = new ConnectionCheckThread(clientInvoker, ir);
  |          t.start();
  |          Thread.sleep(pingTimeout);
  |          pingWorked = t.isValid();
  |       }
  |       catch (Throwable t)
  |       {
  |          log.debug("ConnectionValidator failed to ping via " + clientInvoker, t);
  |       }
  | 
  |       return pingWorked;
  |    }
  |    
  |    private static Map createPingConfig(Map config, Map metadata)
  |    {
  |       Map localConfig = new HashMap();
  |       localConfig.put("connection_checker", "true");
  | 
  |       if (config != null)
  |       {
  |          Object o = config.get(VALIDATOR_PING_TIMEOUT);
  |          log.trace("config timeout: " + o);
  |          if (o != null)
  |          {
  |             try
  |             {
  |                Integer.parseInt((String) o);
  |                localConfig.put(ServerInvoker.TIMEOUT, o);
  |             }
  |             catch (NumberFormatException e)
  |             {
  |                log.warn("Need integer for value of parameter " + VALIDATOR_PING_TIMEOUT + 
  |                         ". Using default value " + DEFAULT_PING_TIMEOUT);
  |             }
  |          }
  |          
  |          o = config.get("NumberOfCallRetries");
  |          if (o != null)
  |          {
  |             localConfig.put("NumberOfCallRetries", o);
  |          }
  |          
  |          o = config.get("NumberOfRetries");
  |          if (o != null)
  |          {
  |             localConfig.put("NumberOfRetries", o);
  |          }
  |       }
  |       
  |       if (metadata != null)
  |       {
  |          metadata.remove(ServerInvoker.TIMEOUT);
  |          localConfig.putAll(metadata);
  |          Object o = metadata.get(VALIDATOR_PING_TIMEOUT);
  |          if (o != null)
  |          {
  |             try
  |             {
  |                Integer.parseInt((String) o);
  |                localConfig.put(ServerInvoker.TIMEOUT, o);
  |             }
  |             catch (NumberFormatException e)
  |             {
  |                log.warn("Need integer for value of parameter " + VALIDATOR_PING_TIMEOUT +
  |                         ". Using default value " + DEFAULT_PING_TIMEOUT);
  |             }
  |          }
  |       }
  |       
  |       if (localConfig.get(ServerInvoker.TIMEOUT) == null)
  |       {
  |          localConfig.put(ServerInvoker.TIMEOUT, DEFAULT_PING_TIMEOUT);
  |       }
  |       
  |       if (localConfig.get("NumberOfCallRetries") == null)
  |       {
  |          localConfig.put("NumberOfCallRetries", DEFAULT_NUMBER_OF_PING_RETRIES);
  |       }
  |       
  |       return localConfig;
  |    }
  | 
  |    // Attributes -----------------------------------------------------------------------------------
  | 
  |    private Client client;
  |    private long pingPeriod;
  |    private Map metadata;
  |    private InvokerLocator locator;
  |    private Map configMap;
  |    private List listeners;
  |    private ClientInvoker clientInvoker;
  |    private Object lock = new Object();
  |    private Object notificationLock = new Object();
  |    private boolean started;
  |    private volatile boolean stopped;
  |    private String invokerSessionId;
  |    private boolean tieToLease = true;
  |    private boolean stopLeaseOnFailure = true;
  |    private int pingTimeout;
  |    private boolean isValid;
  |    private Timer timer;
  | 
  |    volatile TimerTask tt;
  | 
  |    // Constructors ---------------------------------------------------------------------------------
  | 
  |    public ConnectionValidator(Client client)
  |    {
  |       this(client, DEFAULT_PING_PERIOD);
  |    }
  | 
  |    public ConnectionValidator(Client client, long pingPeriod)
  |    {
  |       this.client = client;
  |       this.pingPeriod = pingPeriod;
  |       listeners = new ArrayList();
  |       stopped = false;
  |       getParameters(client, new HashMap());
  |       log.debug(this + " created");
  |    }
  |    
  |    public ConnectionValidator(Client client, Map metadata)
  |    {
  |       this.client = client;
  |       pingPeriod = DEFAULT_PING_PERIOD;
  |       listeners = new ArrayList();
  |       stopped = false;
  |       this.metadata = new HashMap(metadata);
  |       getParameters(client, metadata);
  |       log.debug(this + " created");
  |    }
  | 
  |    // StoppableTimerTask implementation ------------------------------------------------------------
  | 
  |    public void stop()
  |    {
  |       if (stopped)
  |       {
  |          return;
  |       }
  | 
  |       doStop();
  |    }
  | 
  |    // TimerTask overrides --------------------------------------------------------------------------
  | 
  |    /**
  |     * The action to be performed by this timer task.
  |     */
  |    public void run()
  |    {
  |       synchronized (lock) {
  |          if (!started)
  |          {
  |             throw new IllegalStateException(
  |                   ConnectionValidator.class.getName() + ".run() should not be " +
  |                   "called directly; use " + ConnectionValidator.class.getName() +
  |                   ".addConnectionListener() instead.");
  |          }
  |          
  |          if (stopped)
  |          {
  |             return;
  |          }
  |          
  |          while(tt != null) {}
  | 
  |          tt = new WaitOnConnectionCheckTimerTask();
  | 
  |          try
  |          {
  |             timer.schedule(tt, 0);
  |          }
  |          catch (IllegalStateException e)
  |          {
  |             log.debug("Unable to schedule TimerTask on existing Timer", e);
  |             timer = new Timer(true);
  |             timer.schedule(tt, 0);
  |          }
  |       }
  | 
  |       try
  |       {
  |          if(!stopped)
  |          {
  |             isValid = false;
  | 
  |             if (tieToLease && client.getLeasePeriod() > 0)
  |             {
  |                if (trace)
  |                {
  |                   log.trace(this + " sending PING tied to lease");
  |                }
  |                isValid = doCheckConnectionWithLease();
  |             }
  |             else
  |             {
  |                if (trace) { log.trace(this + " pinging ..."); }
  |                isValid = doCheckConnectionWithoutLease();
  |             }
  |          }
  |       }
  |       catch (Throwable thr)
  |       {
  |          log.debug(this + " got throwable while pinging", thr);
  | 
  |          if (stopLeaseOnFailure)
  |          {
  |             log.debug(this + " detected connection failure: stopping");
  |             cancel();
  |          }
  |       }
  |       finally
  |       {
  |          synchronized (notificationLock)
  |          {
  |             notificationLock.notifyAll();
  |          }
  |       }
  |    }
  | 
  |    public boolean cancel()
  |    {
  |       return doStop();
  |    }
  | 
  |    // Public ---------------------------------------------------------------------------------------
  | 
  |    public void addConnectionListener(ConnectionListener listener)
  |    {
  |       if (listener != null)
  |       {
  |          synchronized (lock)
  |          {
  |             if (listeners.size() == 0)
  |             {
  |                start();
  |             }
  |             listeners.add(listener);
  |          }
  |       }
  |    }
  | 
  |    public boolean removeConnectionListener(ConnectionListener listener)
  |    {
  |       boolean isRemoved = false;
  |       if (listener != null)
  |       {
  |          synchronized (lock)
  |          {
  |             isRemoved = listeners.remove(listener);
  |             if (listeners.size() == 0)
  |             {
  |                stop();
  |             }
  |          }
  |       }
  |       return isRemoved;
  |    }
  | 
  |    public long getPingPeriod()
  |    {
  |       if (stopped)
  |       {
  |          return -1;
  |       }
  | 
  |       return pingPeriod;
  |    }
  | 
  |    public String toString()
  |    {
  |       return "ConnectionValidator[" + clientInvoker + ", pingPeriod=" + pingPeriod + " ms]";
  |    }
  | 
  |    // Package protected ----------------------------------------------------------------------------
  | 
  |    // Protected ------------------------------------------------------------------------------------
  | 
  |    // Private --------------------------------------------------------------------------------------
  | 
  |    private void getParameters(Client client, Map metadata)
  |    {
  |       getParametersFromMap(client.getConfiguration());
  |       getParametersFromMap(metadata);
  |       
  |       ClientInvoker clientInvoker = client.getInvoker();
  |       if (clientInvoker instanceof MicroRemoteClientInvoker)
  |       {
  |          invokerSessionId = ((MicroRemoteClientInvoker) clientInvoker).getSessionId();
  |       }
  |       else
  |       {
  |          throw new RuntimeException("creating a ConnectionValidator on a local connection");
  |       }
  |    }
  |    
  |    private void getParametersFromMap(Map config)
  |    {
  |       if (config != null)
  |       {  
  |          Object o = config.get(VALIDATOR_PING_PERIOD);
  |          if (o != null)
  |          {
  |             if (o instanceof String)
  |             {
  |                try 
  |                {
  |                   pingPeriod = Long.parseLong((String)o);
  |                }
  |                catch (Exception e)
  |                {
  |                   log.warn(this + " could not convert " + VALIDATOR_PING_PERIOD +
  |                            " value of " + o + " to a long value");
  |                }
  |             }
  |             else
  |             {
  |                log.warn(this + " could not convert " + VALIDATOR_PING_PERIOD +
  |                         " value of " + o + " to a long value: must be a String");
  |             }
  |          }
  | 
  |          o = config.get(TIE_TO_LEASE);
  |          if (o != null)
  |          {
  |             if (o instanceof String)
  |             {
  |                try
  |                {
  |                   tieToLease = Boolean.valueOf(((String) o)).booleanValue();
  |                }
  |                catch (Exception e)
  |                {
  |                   log.warn(this + " could not convert " + TIE_TO_LEASE + " value" +
  |                         " to a boolean: " + o);
  |                }
  |             }
  |             else
  |             {
  |                log.warn(this + " could not convert " + TIE_TO_LEASE + " value" +
  |                " to a boolean: must be a String");
  |             }
  |          }
  | 
  |          o = config.get(STOP_LEASE_ON_FAILURE);
  |          if (o != null)
  |          {
  |             if (o instanceof String)
  |             {
  |                try
  |                {
  |                   stopLeaseOnFailure = Boolean.valueOf(((String) o)).booleanValue();
  |                }
  |                catch (Exception e)
  |                {
  |                   log.warn(this + " could not convert " + STOP_LEASE_ON_FAILURE + " value" +
  |                         " to a boolean: " + o);
  |                }
  |             }
  |             else
  |             {
  |                log.warn(this + " could not convert " + STOP_LEASE_ON_FAILURE + " value" +
  |                " to a boolean: must be a String");
  |             }
  |          }
  |       }
  |    }
  |    
  |    private void start()
  |    {
  |       configMap = createPingConfig(client.getConfiguration(), metadata);
  |       pingTimeout = Integer.parseInt((String) configMap.get(ServerInvoker.TIMEOUT));
  |       log.debug(this + " timeout: " + pingTimeout);
  |       log.debug(this + " ping retries: " + configMap.get("NumberOfCallRetries"));
  |       log.debug(this + " connection retries: " + configMap.get("NumberOfRetries"));
  |       locator = client.getInvoker().getLocator();
  | 
  |       try
  |       {
  |          clientInvoker = InvokerRegistry.createClientInvoker(locator, configMap);
  |       }
  |       catch (Exception e)
  |       {
  |          log.debug("Unable to create client invoker for locator: " + locator);
  |          throw new RuntimeException("Unable to create client invoker for locator: " + locator, e);
  |       }
  | 
  |       if (!clientInvoker.isConnected())
  |       {
  |          if (trace) { log.trace("inner client invoker not connected, connecting ..."); }
  |          clientInvoker.connect();
  |       }
  | 
  |       TimerUtil.schedule(this, pingPeriod);
  |       started = true;
  |       timer = new Timer(true);
  |       log.debug(this + " started");
  |    }
  |    
  |    private boolean doCheckConnectionWithLease() throws Throwable
  |    {
  |       boolean pingWorked = false;
  | 
  |       try
  |       {
  |          Map metadata = new HashMap();
  |          metadata.put(ServerInvoker.INVOKER_SESSION_ID, invokerSessionId);
  |          InvocationRequest ir =
  |             new InvocationRequest(null, Subsystem.SELF, "$PING$", metadata, null, null);
  | 
  |          if (trace) { log.trace("pinging, sending " + ir + " over " + clientInvoker); }
  | 
  |          Object o = clientInvoker.invoke(ir);
  | 
  |          if (trace) { 
  |             log.trace("Return from server ping is :" + o); 
  | 	 }
  | 
  |          if (o instanceof Boolean && !((Boolean) o).booleanValue())
  |          {
  |             // Server indicates lease has stopped.
  |             throw new Exception();
  |          }
  | 
  |          if (trace) { log.trace("ConnectionValidator got successful ping using " + clientInvoker);}
  | 
  |          pingWorked = true;
  |       }
  |       catch (Throwable t)
  |       {
  |          log.debug("ConnectionValidator failed to ping via " + clientInvoker, t);
  |       }
  | 
  |       return pingWorked;
  |    }
  |    
  |    private boolean doCheckConnectionWithoutLease() throws Throwable
  |    {
  |       boolean pingWorked = false;
  | 
  |       try
  |       {
  |          // Sending null client id as don't want to trigger lease on server side. This also means
  |          // that client connection validator will NOT impact client lease, so can not depend on it
  |          // to maintain client lease with the server.
  |          InvocationRequest ir =
  |             new InvocationRequest(null, Subsystem.SELF, "$PING$", null, null, null);
  | 
  |          if (trace) { log.trace("pinging, sending " + ir + " over " + clientInvoker); }
  | 
  |          clientInvoker.invoke(ir);
  | 
  |          if (trace) { log.trace("ConnectionValidator got successful ping using " + clientInvoker);}
  | 
  |          pingWorked = true;
  |       }
  |       catch (Throwable t)
  |       {
  |          log.debug("ConnectionValidator failed to ping via " + clientInvoker, t);
  |       }
  | 
  |       return pingWorked;
  |    }
  | 
  |    private boolean doStop()
  |    {
  |       synchronized(lock)
  |       {
  |          if (stopped)
  |          {
  |             return false;
  |          }
  |          
  |          if (!listeners.isEmpty())
  |          {
  |             listeners.clear();
  |          }
  |          stopped = true;
  |          timer = null;
  |       }
  | 
  |       if (clientInvoker != null)
  |       {
  |          InvokerRegistry.destroyClientInvoker(locator, configMap);
  |       }
  | 
  |       TimerUtil.unschedule(this);
  | 
  |       boolean result = super.cancel();
  |       log.debug(this + " stopped, returning " + result);
  |       return result;
  |    }
  | 
  |    private void notifyListeners(Throwable thr)
  |    {
  |       final Throwable t = thr;
  |       synchronized (lock)
  |       {
  |          if (stopped)
  |          {
  |             return;
  |          }
  |          ListIterator itr = listeners.listIterator();
  |          while (itr.hasNext())
  |          {
  |             final ConnectionListener listener = (ConnectionListener) itr.next();
  |             new Thread()
  |             {
  |                public void run()
  |                {
  |                   listener.handleConnectionException(t, client);
  |                }
  |             }.start();
  |          }
  |       }
  |       stop();
  |       listeners.clear();
  |    }
  | 
  |    // Inner classes --------------------------------------------------------------------------------
  | 
  |    private class WaitOnConnectionCheckTimerTask extends TimerTask
  |    {
  |       public void run()
  |       {
  |          long start = System.currentTimeMillis();
  | 
  |          synchronized (notificationLock)
  |          {
  |             while (true)
  |             {
  |                int elapsed = (int) (System.currentTimeMillis() - start);
  |                int wait = pingTimeout - elapsed;
  |                if (wait <= 0)
  |                {
  |                   break;
  |                }
  |                
  |                try
  |                {
  |                   notificationLock.wait(wait);
  |                   break;
  |                }
  |                catch (InterruptedException e)
  |                {
  |                   continue;
  |                }
  |             }
  |          }
  |          
  |          if (!isValid)
  |          {
  |             log.debug(ConnectionValidator.this + "'s connections is invalid");
  | 
  |             notifyListeners(new Exception("Could not connect to server!"));
  |             
  |             if (stopLeaseOnFailure)
  |             {
  |                log.debug(this + " detected connection failure: stopping LeasePinger");
  |                MicroRemoteClientInvoker invoker = (MicroRemoteClientInvoker) client.getInvoker();
  |                
  |                if (invoker != null)
  |                {
  |                   invoker.terminateLease(null, client.getDisconnectTimeout());
  |                   log.debug(ConnectionValidator.this + " shut down lease pinger");
  |                }
  |                else
  |                {
  |                   log.debug(ConnectionValidator.this + " unable to shut down lease pinger: client must have shut down");
  |                }
  |                
  |                cancel();
  |             }
  |          }
  | 
  |          tt = null;
  |       }
  |    }
  |    
  |    private static class ConnectionCheckThread extends Thread
  |    {
  |       private InvocationRequest ir;
  |       private ClientInvoker clientInvoker;
  |       private boolean isValid;
  | 
  |       public ConnectionCheckThread(ClientInvoker clientInvoker, InvocationRequest ir)
  |       {
  |          this.clientInvoker = clientInvoker;
  |          this.ir = ir;
  |          setDaemon(true);
  |       }
  |       
  |       public void run()
  |       {
  |          try
  |          {
  |             if (trace) { log.trace("pinging, sending " + ir + " over " + clientInvoker); }
  |             clientInvoker.invoke(ir);
  |             isValid = true;
  |             if (trace) { log.trace("ConnectionValidator got successful ping using " + clientInvoker);}
  |          }
  |          catch (Throwable t)
  |          {
  |             log.debug("ConnectionValidator failed to ping via " + clientInvoker, t);
  |          }
  |       }
  |       
  |       public boolean isValid()
  |       {
  |          return isValid;
  |       }
  |    }
  | }
  | 

View the original post : http://www.jboss.org/index.html?module=bb&op=viewtopic&p=4214135#4214135

Reply to the post : http://www.jboss.org/index.html?module=bb&op=posting&mode=reply&p=4214135



More information about the jboss-user mailing list