Log instance creation in class AbstractLocalCacheStream & AbstractCacheStream, cause system hang / hit performance ------------------------------------------------------------------------------------------------------------------ Key: ISPN-8647 URL: https://issues.jboss.org/browse/ISPN-8647 Project: Infinispan Issue Type: Bug Components: Core Affects Versions: 9.1.1.Final, 9.1.2.Final, 9.1.3.Final, 9.1.4.Final Environment: On production environment Reporter: paragBorah Borah Priority: Critical Attachments: TD1(1).zip Log instance creation got blocked in class AbstractLocalCacheStream & AbstractCacheStream, while performing stream operations on infinispan. On each thread dump, getting below exception multiple times. Which hang the system & hit performance "Thread-93,EXECUTOR_POOL" #57030 daemon prio=5 os_prio=0 tid=0x00000000867ca000 nid=0x2a78 waiting for monitor entry [0x00000000d349d000] java.lang.Thread.State: BLOCKED (on object monitor) at java.lang.ClassLoader.loadClass(ClassLoader.java:404) waiting to lock <0x00000002cdb18a78> (a java.lang.Object) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.jboss.logging.Logger$1.run(Logger.java:2544) at java.security.AccessController.doPrivileged(Native Method) at org.jboss.logging.Logger.getMessageLogger(Logger.java:2529) at org.jboss.logging.Logger.getMessageLogger(Logger.java:2516) at org.infinispan.util.logging.LogFactory.getLog(LogFactory.java:19) at org.infinispan.stream.impl.AbstractCacheStream.<init>(AbstractCacheStream.java:53) at org.infinispan.stream.impl.DistributedCacheStream.<init>(DistributedCacheStream.java:120) at org.infinispan.interceptors.distribution.DistributionBulkInterceptor$BackingEntrySet$1.<init>(DistributionBulkInterceptor.java:147) at org.infinispan.interceptors.distribution.DistributionBulkInterceptor$BackingEntrySet.stream(DistributionBulkInterceptor.java:147) at org.infinispan.interceptors.distribution.DistributionBulkInterceptor$BackingEntrySet.iterator(DistributionBulkInterceptor.java:102) at org.infinispan.stream.impl.local.ValueCacheCollection.iterator(ValueCacheCollection.java:38) at org.infinispan.cache.impl.EncoderCache$EncoderValuesCollection.iterator(EncoderCache.java:873) at org.infinispan.cache.impl.EncoderCache$EncoderValuesCollection.iterator(EncoderCache.java:852) at com.bmc.unifiedadmin.persistence.service.server.ServerDetailsCache.getIntegrationServiceById(ServerDetailsCache.java:788) at com.bmc.unifiedadmin.persistence.service.server.ServerDetailsCache.getProxyDetails(ServerDetailsCache.java:419) at com.bmc.unifiedadmin.persistence.service.server.IntegrationServicePersistenceServiceImpl.getIntegrationServiceDetails(IntegrationServicePersistenceServiceImpl.java:138) at com.bmc.unifiedadmin.persistence.service.server.IntegrationServicePersistenceServiceImpl.getAssociatedPAsWithISs(IntegrationServicePersistenceServiceImpl.java:828) at com.bmc.sas.ngp.service.unifiedadmin.integrationservice.IntegrationServiceImpl.getAllProxiesForServer(IntegrationServiceImpl.java:226) at com.proactivenet.api.aspen.ws.CAWSFacadePlatformImpl.loadServerDetails(CAWSFacadePlatformImpl.java:1255) at com.proactivenet.api.aspen.ws.CAWSFacadePlatformImpl.getAllServerAndISDetails(CAWSFacadePlatformImpl.java:1283) at com.bmc.sms.cmaws.ngp.wsclient.CMAJServiceClient.getAllServerAndISDetails(CMAJServiceClient.java:452) at com.bmc.sms.cmaws.rsbuilder.CMATSIMServerDetailsResultBuilder.buildResult(CMATSIMServerDetailsResultBuilder.java:75) at com.bmc.truesight.rest.taskexecution.TaskExecutionWorker.run(TaskExecutionWorker.java:83) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) == "GatewayEvtMessageProcessor" #185 prio=5 os_prio=0 tid=0x0000000074094000 nid=0x2454 waiting for monitor entry [0x000000007fa8d000] java.lang.Thread.State: BLOCKED (on object monitor) at java.lang.ClassLoader.loadClass(ClassLoader.java:404) waiting to lock <0x00000002cdb18a78> (a java.lang.Object) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.jboss.logging.Logger$1.run(Logger.java:2544) at java.security.AccessController.doPrivileged(Native Method) at org.jboss.logging.Logger.getMessageLogger(Logger.java:2529) at org.jboss.logging.Logger.getMessageLogger(Logger.java:2516) at org.infinispan.util.logging.LogFactory.getLog(LogFactory.java:19) at org.infinispan.stream.impl.local.AbstractLocalCacheStream.<init>(AbstractLocalCacheStream.java:22) at org.infinispan.stream.impl.local.LocalCacheStream.<init>(LocalCacheStream.java:69) at org.infinispan.commands.read.EntrySetCommand$BackingEntrySet.stream(EntrySetCommand.java:145) at org.infinispan.stream.impl.local.ValueCacheCollection.stream(ValueCacheCollection.java:93) at org.infinispan.cache.impl.EncoderCache$EncoderValuesCollection.stream(EncoderCache.java:863) at com.bmc.truesight.api.extended.component.ComponentsDAO.getComponentsByType(ComponentsDAO.java:69) at com.bmc.truesight.api.extended.event.Events.getIdforComponentByCellName(Events.java:2414) at com.bmc.truesight.platform.components.gateway.CellGatewayServer.processNewEvent(CellGatewayServer.java:148) locked <0x00000002c7ad2a58> (a com.bmc.truesight.platform.components.gateway.CellGatewayServer) at com.bmc.sms.cellcomm.gw.GatewayServer$EventMessageProcessor.processMessage(GatewayServer.java:430) at com.bmc.sms.cellcomm.gw.GatewayServer$EventMessageProcessor.run(GatewayServer.java:340) at java.lang.Thread.run(Thread.java:748)