Author: remy.maucherat(a)jboss.com Date: 2012-01-30 10:17:31 -0500 (Mon, 30 Jan 2012) New Revision: 1939 Added: trunk/java/org/apache/catalina/filters/SetCharacterEncodingFilter.java trunk/java/org/apache/catalina/valves/CrawlerSessionManagerValve.java trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java Modified: trunk/webapps/docs/changelog.xml Log: Not very useful, but port some valves and filters. Added: trunk/java/org/apache/catalina/filters/SetCharacterEncodingFilter.java =================================================================== --- trunk/java/org/apache/catalina/filters/SetCharacterEncodingFilter.java (rev 0) +++ trunk/java/org/apache/catalina/filters/SetCharacterEncodingFilter.java 2012-01-30 15:17:31 UTC (rev 1939) @@ -0,0 +1,122 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.catalina.filters; + +import java.io.IOException; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; + + +/** + * <p>Example filter that sets the character encoding to be used in parsing the + * incoming request, either unconditionally or only if the client did not + * specify a character encoding. Configuration of this filter is based on + * the following initialization parameters:</p> + * <ul> + * <li><strong>encoding</strong> - The character encoding to be configured + * for this request, either conditionally or unconditionally based on + * the <code>ignore</code> initialization parameter. This parameter + * is required, so there is no default.</li> + * <li><strong>ignore</strong> - If set to "true", any character encoding + * specified by the client is ignored, and the value returned by the + * <code>selectEncoding()</code> method is set. If set to "false, + * <code>selectEncoding()</code> is called <strong>only</strong> if the + * client has not already specified an encoding. By default, this + * parameter is set to "false".</li> + * </ul> + * + * <p>Although this filter can be used unchanged, it is also easy to + * subclass it and make the <code>selectEncoding()</code> method more + * intelligent about what encoding to choose, based on characteristics of + * the incoming request (such as the values of the <code>Accept-Language</code> + * and <code>User-Agent</code> headers, or a value stashed in the current + * user's session.</p> + */ +public class SetCharacterEncodingFilter extends FilterBase { + + // ----------------------------------------------------- Instance Variables + + /** + * The default character encoding to set for requests that pass through + * this filter. + */ + private String encoding = null; + public void setEncoding(String encoding) { this.encoding = encoding; } + public String getEncoding() { return encoding; } + + + /** + * Should a character encoding specified by the client be ignored? + */ + private boolean ignore = false; + public void setIgnore(boolean ignore) { this.ignore = ignore; } + public boolean isIgnore() { return ignore; } + + + // --------------------------------------------------------- Public Methods + + + /** + * Select and set (if specified) the character encoding to be used to + * interpret request parameters for this request. + * + * @param request The servlet request we are processing + * @param response The servlet response we are creating + * @param chain The filter chain we are processing + * + * @exception IOException if an input/output error occurs + * @exception ServletException if a servlet error occurs + */ + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain chain) + throws IOException, ServletException { + + // Conditionally select and set the character encoding to be used + if (ignore || (request.getCharacterEncoding() == null)) { + String characterEncoding = selectEncoding(request); + if (characterEncoding != null) { + request.setCharacterEncoding(characterEncoding); + } + } + + // Pass control on to the next filter + chain.doFilter(request, response); + } + + + // ------------------------------------------------------ Protected Methods + + /** + * Select an appropriate character encoding to be used, based on the + * characteristics of the current request and/or filter initialization + * parameters. If no character encoding should be set, return + * <code>null</code>. + * <p> + * The default implementation unconditionally returns the value configured + * by the <strong>encoding</strong> initialization parameter for this + * filter. + * + * @param request The servlet request we are processing + */ + protected String selectEncoding(ServletRequest request) { + return this.encoding; + } +} Added: trunk/java/org/apache/catalina/valves/CrawlerSessionManagerValve.java =================================================================== --- trunk/java/org/apache/catalina/valves/CrawlerSessionManagerValve.java (rev 0) +++ trunk/java/org/apache/catalina/valves/CrawlerSessionManagerValve.java 2012-01-30 15:17:31 UTC (rev 1939) @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.catalina.valves; + +import java.io.IOException; +import java.util.Enumeration; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpSession; +import javax.servlet.http.HttpSessionBindingEvent; +import javax.servlet.http.HttpSessionBindingListener; + +import org.apache.catalina.Lifecycle; +import org.apache.catalina.LifecycleException; +import org.apache.catalina.LifecycleListener; +import org.apache.catalina.connector.Request; +import org.apache.catalina.connector.Response; +import org.apache.catalina.util.LifecycleSupport; +import org.jboss.logging.Logger; + +/** + * Web crawlers can trigger the creation of many thousands of sessions as they + * crawl a site which may result in significant memory consumption. This Valve + * ensures that crawlers are associated with a single session - just like normal + * users - regardless of whether or not they provide a session token with their + * requests. + */ +public class CrawlerSessionManagerValve extends ValveBase + implements Lifecycle, HttpSessionBindingListener { + + private static Logger log = Logger.getLogger(CrawlerSessionManagerValve.class); + + /** + * The lifecycle event support for this component. + */ + protected LifecycleSupport lifecycle = new LifecycleSupport(this); + protected boolean started = false; + + private final Map<String,String> clientIpSessionId = + new ConcurrentHashMap<String, String>(); + private final Map<String,String> sessionIdClientIp = + new ConcurrentHashMap<String, String>(); + + private String crawlerUserAgents = + ".*[bB]ot.*|.*Yahoo! Slurp.*|.*Feedfetcher-Google.*"; + private Pattern uaPattern = null; + private int sessionInactiveInterval = 60; + + + /** + * Specify the regular expression (using {@link Pattern}) that will be used + * to identify crawlers based in the User-Agent header provided. The default + * is ".*GoogleBot.*|.*bingbot.*|.*Yahoo! Slurp.*" + * + * @param crawlerUserAgents The regular expression using {@link Pattern} + */ + public void setCrawlerUserAgents(String crawlerUserAgents) { + this.crawlerUserAgents = crawlerUserAgents; + if (crawlerUserAgents == null || crawlerUserAgents.length() == 0) { + uaPattern = null; + } else { + uaPattern = Pattern.compile(crawlerUserAgents); + } + } + + /** + * @see #setCrawlerUserAgents(String) + * @return The current regular expression being used to match user agents. + */ + public String getCrawlerUserAgents() { + return crawlerUserAgents; + } + + + /** + * Specify the session timeout (in seconds) for a crawler's session. This is + * typically lower than that for a user session. The default is 60 seconds. + * + * @param sessionInactiveInterval The new timeout for crawler sessions + */ + public void setSessionInactiveInterval(int sessionInactiveInterval) { + this.sessionInactiveInterval = sessionInactiveInterval; + } + + /** + * @see #setSessionInactiveInterval(int) + * @return The current timeout in seconds + */ + public int getSessionInactiveInterval() { + return sessionInactiveInterval; + } + + + public Map<String,String> getClientIpSessionId() { + return clientIpSessionId; + } + + + // ------------------------------------------------------ Lifecycle Methods + + + public void addLifecycleListener(LifecycleListener listener) { + lifecycle.addLifecycleListener(listener); + } + + + public LifecycleListener[] findLifecycleListeners() { + return lifecycle.findLifecycleListeners(); + } + + + public void removeLifecycleListener(LifecycleListener listener) { + lifecycle.removeLifecycleListener(listener); + } + + + public void start() throws LifecycleException { + + // Validate and update our current component state + if (started) + throw new LifecycleException(sm + .getString("accessLogValve.alreadyStarted")); + lifecycle.fireLifecycleEvent(START_EVENT, null); + started = true; + + uaPattern = Pattern.compile(crawlerUserAgents); + } + + + public void stop() throws LifecycleException { + + // Validate and update our current component state + if (!started) + throw new LifecycleException(sm + .getString("accessLogValve.notStarted")); + lifecycle.fireLifecycleEvent(STOP_EVENT, null); + started = false; + + } + + + @Override + public void invoke(Request request, Response response) throws IOException, + ServletException { + + boolean isBot = false; + String sessionId = null; + String clientIp = null; + + if (log.isDebugEnabled()) { + log.debug(request.hashCode() + ": ClientIp=" + + request.getRemoteAddr() + ", RequestedSessionId=" + + request.getRequestedSessionId()); + } + + // If the incoming request has a valid session ID, no action is required + if (request.getSession(false) == null) { + + // Is this a crawler - check the UA headers + Enumeration<String> uaHeaders = request.getHeaders("user-agent"); + String uaHeader = null; + if (uaHeaders.hasMoreElements()) { + uaHeader = uaHeaders.nextElement(); + } + + // If more than one UA header - assume not a bot + if (uaHeader != null && !uaHeaders.hasMoreElements()) { + + if (log.isDebugEnabled()) { + log.debug(request.hashCode() + ": UserAgent=" + uaHeader); + } + + if (uaPattern.matcher(uaHeader).matches()) { + isBot = true; + + if (log.isDebugEnabled()) { + log.debug(request.hashCode() + + ": Bot found. UserAgent=" + uaHeader); + } + } + } + + // If this is a bot, is the session ID known? + if (isBot) { + clientIp = request.getRemoteAddr(); + sessionId = clientIpSessionId.get(clientIp); + if (sessionId != null) { + request.setRequestedSessionId(sessionId); + if (log.isDebugEnabled()) { + log.debug(request.hashCode() + ": SessionID=" + + sessionId); + } + } + } + } + + getNext().invoke(request, response); + + if (isBot) { + if (sessionId == null) { + // Has bot just created a session, if so make a note of it + HttpSession s = request.getSession(false); + if (s != null) { + clientIpSessionId.put(clientIp, s.getId()); + sessionIdClientIp.put(s.getId(), clientIp); + // #valueUnbound() will be called on session expiration + s.setAttribute(this.getClass().getName(), this); + s.setMaxInactiveInterval(sessionInactiveInterval); + + if (log.isDebugEnabled()) { + log.debug(request.hashCode() + + ": New bot session. SessionID=" + s.getId()); + } + } + } else { + if (log.isDebugEnabled()) { + log.debug(request.hashCode() + + ": Bot session accessed. SessionID=" + sessionId); + } + } + } + } + + + @Override + public void valueBound(HttpSessionBindingEvent event) { + // NOOP + } + + + @Override + public void valueUnbound(HttpSessionBindingEvent event) { + String clientIp = sessionIdClientIp.remove(event.getSession().getId()); + if (clientIp != null) { + clientIpSessionId.remove(clientIp); + } + } +} Added: trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java =================================================================== --- trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java (rev 0) +++ trunk/java/org/apache/catalina/valves/StuckThreadDetectionValve.java 2012-01-30 15:17:31 UTC (rev 1939) @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.catalina.valves; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.servlet.ServletException; + +import org.apache.catalina.connector.Request; +import org.apache.catalina.connector.Response; +import org.apache.tomcat.util.res.StringManager; +import org.jboss.logging.Logger; +import org.jboss.servlet.http.HttpEvent; + +/** + * This valve allows to detect requests that take a long time to process, which might + * indicate that the thread that is processing it is stuck. + * Based on code proposed by TomLu in Bugzilla entry #50306 + * + * @author slaurent + * + */ +public class StuckThreadDetectionValve extends ValveBase { + + /** + * Logger + */ + private static Logger log = Logger.getLogger(StuckThreadDetectionValve.class); + + /** + * The string manager for this package. + */ + private static final StringManager sm = + StringManager.getManager(Constants.Package); + + /** + * Keeps count of the number of stuck threads detected + */ + private final AtomicInteger stuckCount = new AtomicInteger(0); + + /** + * In seconds. Default 600 (10 minutes). + */ + private int threshold = 600; + + /** + * The only references we keep to actual running Thread objects are in + * this Map (which is automatically cleaned in invoke()s finally clause). + * That way, Threads can be GC'ed, eventhough the Valve still thinks they + * are stuck (caused by a long monitor interval) + */ + private final ConcurrentHashMap<Long, MonitoredThread> activeThreads = + new ConcurrentHashMap<Long, MonitoredThread>(); + /** + * + */ + private final Queue<CompletedStuckThread> completedStuckThreadsQueue = + new ConcurrentLinkedQueue<CompletedStuckThread>(); + + /** + * Specify the threshold (in seconds) used when checking for stuck threads. + * If &lt;=0, the detection is disabled. The default is 600 seconds. + * + * @param threshold + * The new threshold in seconds + */ + public void setThreshold(int threshold) { + this.threshold = threshold; + } + + /** + * @see #setThreshold(int) + * @return The current threshold in seconds + */ + public int getThreshold() { + return threshold; + } + + + private void notifyStuckThreadDetected(MonitoredThread monitoredThread, + long activeTime, int numStuckThreads) { + String msg = sm.getString( + "stuckThreadDetectionValve.notifyStuckThreadDetected", + monitoredThread.getThread().getName(), Long.valueOf(activeTime), + monitoredThread.getStartTime(), + Integer.valueOf(numStuckThreads), + monitoredThread.getRequestUri(), Integer.valueOf(threshold)); + // msg += "\n" + getStackTraceAsString(trace); + Throwable th = new Throwable(); + th.setStackTrace(monitoredThread.getThread().getStackTrace()); + log.warn(msg, th); + } + + private void notifyStuckThreadCompleted(String threadName, + long activeTime, int numStuckThreads) { + String msg = sm.getString( + "stuckThreadDetectionValve.notifyStuckThreadCompleted", + threadName, Long.valueOf(activeTime), + Integer.valueOf(numStuckThreads)); + // Since the "stuck thread notification" is warn, this should also + // be warn + log.warn(msg); + } + + /** + * {@inheritDoc} + */ + @Override + public void invoke(Request request, Response response) + throws IOException, ServletException { + + if (threshold <= 0) { + // short-circuit if not monitoring stuck threads + getNext().invoke(request, response); + return; + } + + // Save the thread/runnable + // Keeping a reference to the thread object here does not prevent + // GC'ing, as the reference is removed from the Map in the finally clause + + Long key = Long.valueOf(Thread.currentThread().getId()); + StringBuffer requestUrl = request.getRequestURL(); + if(request.getQueryString()!=null) { + requestUrl.append("?"); + requestUrl.append(request.getQueryString()); + } + MonitoredThread monitoredThread = new MonitoredThread(Thread.currentThread(), + requestUrl.toString()); + activeThreads.put(key, monitoredThread); + + try { + getNext().invoke(request, response); + } finally { + activeThreads.remove(key); + if (monitoredThread.markAsDone() == MonitoredThreadState.STUCK) { + completedStuckThreadsQueue.add( + new CompletedStuckThread(monitoredThread.getThread().getName(), + monitoredThread.getActiveTimeInMillis())); + } + } + } + + @Override + public void event(Request request, Response response, HttpEvent event) + throws IOException, ServletException { + + if (threshold <= 0) { + // short-circuit if not monitoring stuck threads + getNext().event(request, response, event); + return; + } + + // Save the thread/runnable + // Keeping a reference to the thread object here does not prevent + // GC'ing, as the reference is removed from the Map in the finally clause + + Long key = Long.valueOf(Thread.currentThread().getId()); + StringBuffer requestUrl = request.getRequestURL(); + if(request.getQueryString()!=null) { + requestUrl.append("?"); + requestUrl.append(request.getQueryString()); + } + MonitoredThread monitoredThread = new MonitoredThread(Thread.currentThread(), + requestUrl.toString()); + activeThreads.put(key, monitoredThread); + + try { + getNext().event(request, response, event); + } finally { + activeThreads.remove(key); + if (monitoredThread.markAsDone() == MonitoredThreadState.STUCK) { + completedStuckThreadsQueue.add( + new CompletedStuckThread(monitoredThread.getThread().getName(), + monitoredThread.getActiveTimeInMillis())); + } + } + } + + @Override + public void backgroundProcess() { + super.backgroundProcess(); + + long thresholdInMillis = threshold * 1000; + + // Check monitored threads, being careful that the request might have + // completed by the time we examine it + for (MonitoredThread monitoredThread : activeThreads.values()) { + long activeTime = monitoredThread.getActiveTimeInMillis(); + + if (activeTime >= thresholdInMillis && monitoredThread.markAsStuckIfStillRunning()) { + int numStuckThreads = stuckCount.incrementAndGet(); + notifyStuckThreadDetected(monitoredThread, activeTime, numStuckThreads); + } + } + // Check if any threads previously reported as stuck, have finished. + for (CompletedStuckThread completedStuckThread = completedStuckThreadsQueue.poll(); + completedStuckThread != null; completedStuckThread = completedStuckThreadsQueue.poll()) { + + int numStuckThreads = stuckCount.decrementAndGet(); + notifyStuckThreadCompleted(completedStuckThread.getName(), + completedStuckThread.getTotalActiveTime(), numStuckThreads); + } + } + + public long[] getStuckThreadIds() { + List<Long> idList = new ArrayList<Long>(); + for (MonitoredThread monitoredThread : activeThreads.values()) { + if (monitoredThread.isMarkedAsStuck()) { + idList.add(Long.valueOf(monitoredThread.getThread().getId())); + } + } + + long[] result = new long[idList.size()]; + for (int i = 0; i < result.length; i++) { + result[i] = idList.get(i).longValue(); + } + return result; + } + + private static class MonitoredThread { + + /** + * Reference to the thread to get a stack trace from background task + */ + private final Thread thread; + private final String requestUri; + private final long start; + private final AtomicInteger state = new AtomicInteger( + MonitoredThreadState.RUNNING.ordinal()); + + public MonitoredThread(Thread thread, String requestUri) { + this.thread = thread; + this.requestUri = requestUri; + this.start = System.currentTimeMillis(); + } + + public Thread getThread() { + return this.thread; + } + + public String getRequestUri() { + return requestUri; + } + + public long getActiveTimeInMillis() { + return System.currentTimeMillis() - start; + } + + public Date getStartTime() { + return new Date(start); + } + + public boolean markAsStuckIfStillRunning() { + return this.state.compareAndSet(MonitoredThreadState.RUNNING.ordinal(), + MonitoredThreadState.STUCK.ordinal()); + } + + public MonitoredThreadState markAsDone() { + int val = this.state.getAndSet(MonitoredThreadState.DONE.ordinal()); + return MonitoredThreadState.values()[val]; + } + + boolean isMarkedAsStuck() { + return this.state.get() == MonitoredThreadState.STUCK.ordinal(); + } + } + + private static class CompletedStuckThread { + + private final String threadName; + private final long totalActiveTime; + + public CompletedStuckThread(String threadName, long totalActiveTime) { + this.threadName = threadName; + this.totalActiveTime = totalActiveTime; + } + + public String getName() { + return this.threadName; + } + + public long getTotalActiveTime() { + return this.totalActiveTime; + } + } + + private enum MonitoredThreadState { + RUNNING, STUCK, DONE; + } +} Modified: trunk/webapps/docs/changelog.xml =================================================================== --- trunk/webapps/docs/changelog.xml 2012-01-26 16:50:41 UTC (rev 1938) +++ trunk/webapps/docs/changelog.xml 2012-01-30 15:17:31 UTC (rev 1939) @@ -17,6 +17,13 @@ <body> <section name="JBoss Web 7.0.10.Final (remm)"> + <subsection name="Catalina"> + <changelog> + <fix> + Port some additional valves and filters. (remm) + </fix> + </changelog> + </subsection> <subsection name="Jasper"> <changelog> <fix>