ConcurrentModificationException on Resource.childResources during response rendering ------------------------------------------------------------------------------------- Key: EMBJOPR-124 URL: https://jira.jboss.org/browse/EMBJOPR-124 Project: Embedded Jopr Issue Type: Bug Components: Core Infrastructure Affects Versions: 1.1 Reporter: Ian Springer Assignee: Ian Springer Priority: Critical Fix For: 1.2 I got the below stack trace on a Facelets error page immediately after logging into the Admin Console: 13:39:11,991 ERROR [viewhandler] Error Rendering View[/secure/resourceTypeSummary.xhtml] java.util.ConcurrentModificationException at java.util.LinkedHashMap$LinkedHashIterator.nextEntry(LinkedHashMap.java:373) at java.util.LinkedHashMap$KeyIterator.next(LinkedHashMap.java:384) at java.util.HashSet.writeObject(HashSet.java:267) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:945) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1461) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150) at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326) at org.rhq.core.domain.resource.Resource.writeExternal(Resource.java:1509) at java.io.ObjectOutputStream.writeExternalData(ObjectOutputStream.java:1421) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1390) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150) at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326) at org.rhq.core.domain.resource.Resource.writeExternal(Resource.java:1494) at java.io.ObjectOutputStream.writeExternalData(ObjectOutputStream.java:1421) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1390) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1509) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1474) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150) at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326) at java.util.ArrayList.writeObject(ArrayList.java:570) at sun.reflect.GeneratedMethodAccessor359.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:945) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1461) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150) at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326) at java.util.HashMap.writeObject(HashMap.java:1001) at sun.reflect.GeneratedMethodAccessor301.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:945) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1461) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1392) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1150) at java.io.ObjectOutputStream.writeArray(ObjectOutputStream.java:1338) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1146) at java.io.ObjectOutputStream.writeArray(ObjectOutputStream.java:1338) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1146) at java.io.ObjectOutputStream.writeArray(ObjectOutputStream.java:1338) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1146) at java.io.ObjectOutputStream.writeArray(ObjectOutputStream.java:1338) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1146) at java.io.ObjectOutputStream.writeArray(ObjectOutputStream.java:1338) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1146) at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:326) at com.sun.faces.renderkit.ResponseStateManagerImpl.writeState(ResponseStateManagerImpl.java:269) at javax.faces.render.ResponseStateManager.writeState(ResponseStateManager.java:166) at org.ajax4jsf.application.AjaxStateManager.writeState(AjaxStateManager.java:294) at org.ajax4jsf.application.AjaxStateManager.writeState(AjaxStateManager.java:256) at com.sun.facelets.FaceletViewHandler.renderView(FaceletViewHandler.java:620) at org.ajax4jsf.application.ViewHandlerWrapper.renderView(ViewHandlerWrapper.java:100) at org.ajax4jsf.application.AjaxViewHandler.renderView(AjaxViewHandler.java:176) at com.sun.faces.lifecycle.RenderResponsePhase.execute(RenderResponsePhase.java:110) at com.sun.faces.lifecycle.Phase.doPhase(Phase.java:100) at com.sun.faces.lifecycle.LifecycleImpl.render(LifecycleImpl.java:139) at javax.faces.webapp.FacesServlet.service(FacesServlet.java:266) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.jboss.seam.servlet.SeamFilter$FilterChainImpl.doFilter(SeamFilter.java:83) at org.jboss.seam.web.IdentityFilter.doFilter(IdentityFilter.java:38) at org.jboss.seam.servlet.SeamFilter$FilterChainImpl.doFilter(SeamFilter.java:69) at org.jboss.seam.web.MultipartFilter.doFilter(MultipartFilter.java:90) at org.jboss.seam.servlet.SeamFilter$FilterChainImpl.doFilter(SeamFilter.java:69) at org.jboss.seam.web.ExceptionFilter.doFilter(ExceptionFilter.java:64) at org.jboss.seam.servlet.SeamFilter$FilterChainImpl.doFilter(SeamFilter.java:69) at org.jboss.seam.web.RedirectFilter.doFilter(RedirectFilter.java:45) at org.jboss.seam.servlet.SeamFilter$FilterChainImpl.doFilter(SeamFilter.java:69) at org.ajax4jsf.webapp.BaseXMLFilter.doXmlFilter(BaseXMLFilter.java:178) at org.ajax4jsf.webapp.BaseFilter.handleRequest(BaseFilter.java:290) at org.ajax4jsf.webapp.BaseFilter.processUploadsAndHandleRequest(BaseFilter.java:390) at org.ajax4jsf.webapp.BaseFilter.doFilter(BaseFilter.java:517) at org.jboss.seam.web.Ajax4jsfFilter.doFilter(Ajax4jsfFilter.java:56) at org.jboss.seam.servlet.SeamFilter$FilterChainImpl.doFilter(SeamFilter.java:69) at org.jboss.seam.web.LoggingFilter.doFilter(LoggingFilter.java:58) at org.jboss.seam.servlet.SeamFilter$FilterChainImpl.doFilter(SeamFilter.java:69) at org.jboss.seam.debug.hot.HotDeployFilter.doFilter(HotDeployFilter.java:51) at org.jboss.seam.servlet.SeamFilter$FilterChainImpl.doFilter(SeamFilter.java:69) at org.jboss.seam.servlet.SeamFilter.doFilter(SeamFilter.java:158) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFilter.java:96) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:235) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191) at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(SecurityAssociationValve.java:190) at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValve.java:92) at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.process(SecurityContextEstablishmentValve.java:126) at org.jboss.web.tomcat.security.SecurityContextEstablishmentValve.invoke(SecurityContextEstablishmentValve.java:70) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102) at org.jboss.web.tomcat.service.jca.CachedConnectionValve.invoke(CachedConnectionValve.java:158) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:330) at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:829) at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:601) at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:447) at java.lang.Thread.run(Thread.java:619) I've only seen this once, so I think it's fairly rare. It looks to me like the fix would be changing: if (childResources != null && childResources.getClass().getName().contains("hibernate")) { out.writeObject(new LinkedHashSet<Resource>(childResources)); } else { out.writeObject(childResources); } in Resource.writeExternal() to: out.writeObject(new LinkedHashSet<Resource>(childResources)); to eliminate the chance of CMEs when some other thread tries to modify the childResources Set at the same time out.writeObject() is iterating it.