Dear All, We have currently using keycloak 3.4.3 version and trying to migrate to 4.3.0 but the startup is failing due to the migration issue. We have around 400-500 realms in the database. Please find below exception. From the log it is clear that "org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51)" is the one causing the exception (see the code below. Line 51 is in bold). Is this because of large realms? any ideas? @Override public void migrate(KeycloakSession session) { *session.realms().getRealms().stream().forEach(* r -> { migrateRealm(session, r, false); } ); } 22:16:17,002 WARN [com.arjuna.ats.arjuna] (Transaction Reaper) ARJUNA012117: TransactionReaper::check timeout for TX 0:ffffac110004:-14e6f320:5ba958b2:12 in state RUN 22:16:17,070 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012121: TransactionReaper::doCancellations worker Thread[Transaction Reaper Worker 0,5,main] succ essfully canceled TX 0:ffffac110004:-14e6f320:5ba958b2:f 22:16:17,073 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012095: Abort of action id 0:ffffac110004:-14e6f320:5ba958b2:12 invoked while multiple threads ac tive within it. 22:16:17,079 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012381: Action id 0:ffffac110004:-14e6f320:5ba958b2:12 completed with multiple threads - thread S erverService Thread Pool -- 53 was in progress with org.hibernate.event.internal.DefaultPersistEventListener.entityIsPersistent(DefaultPersistEventListener.java:163) org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:128) org.hibernate.internal.SessionImpl.firePersistOnFlush(SessionImpl.java:805) org.hibernate.internal.SessionImpl.persistOnFlush(SessionImpl.java:798) org.hibernate.engine.spi.CascadingActions$8.cascade(CascadingActions.java:340) org.hibernate.engine.internal.Cascade.cascadeToOne(Cascade.java:423) org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:348) org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) org.hibernate.engine.internal.Cascade.cascadeCollectionElements(Cascade.java:456) org.hibernate.engine.internal.Cascade.cascadeCollection(Cascade.java:388) org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:351) org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) org.hibernate.engine.internal.Cascade.cascade(Cascade.java:136) org.hibernate.event.internal.AbstractSaveEventListener.cascadeAfterSave(AbstractSaveEventListener.java:445) org.hibernate.event.internal.DefaultPersistEventListener.justCascade(DefaultPersistEventListener.java:172) org.hibernate.event.internal.DefaultPersistEventListener.entityIsPersistent(DefaultPersistEventListener.java:164) org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:128) org.hibernate.internal.SessionImpl.firePersistOnFlush(SessionImpl.java:805) org.hibernate.internal.SessionImpl.persistOnFlush(SessionImpl.java:798) org.hibernate.engine.spi.CascadingActions$8.cascade(CascadingActions.java:340) org.hibernate.engine.internal.Cascade.cascadeToOne(Cascade.java:423) org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:348) org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) org.hibernate.engine.internal.Cascade.cascadeCollectionElements(Cascade.java:456) org.hibernate.engine.internal.Cascade.cascadeCollection(Cascade.java:388) org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:351) org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) org.hibernate.engine.internal.Cascade.cascade(Cascade.java:136) org.hibernate.event.internal.AbstractFlushingEventListener.cascadeOnFlush(AbstractFlushingEventListener.java:150) org.hibernate.event.internal.AbstractFlushingEventListener.prepareEntityFlushes(AbstractFlushingEventListener.java:141) org.hibernate.event.internal.AbstractFlushingEventListener.flushEverythingToExecutions(AbstractFlushingEventListener.java:74) org.hibernate.event.internal.DefaultAutoFlushEventListener.onAutoFlush(DefaultAutoFlushEventListener.java:44) org.hibernate.internal.SessionImpl.autoFlushIfRequired(SessionImpl.java:1264) org.hibernate.internal.SessionImpl.list(SessionImpl.java:1332) org.hibernate.internal.QueryImpl.list(QueryImpl.java:87) org.hibernate.jpa.internal.QueryImpl.list(QueryImpl.java:606) org.hibernate.jpa.internal.QueryImpl.getResultList(QueryImpl.java:483) org.keycloak.models.jpa.ClientAdapter.getClientScopes(ClientAdapter.java:353) org.keycloak.models.cache.infinispan.entities.CachedClient.<init>(CachedClient.java:119) org.keycloak.models.cache.infinispan.RealmCacheSession.cacheClient(RealmCacheSession.java:1069) org.keycloak.models.cache.infinispan.RealmCacheSession.getClientById(RealmCacheSession.java:1029) org.keycloak.models.jpa.RealmAdapter.getMasterAdminClient(RealmAdapter.java:1037) org.keycloak.models.cache.infinispan.entities.CachedRealm.<init>(CachedRealm.java:235) org.keycloak.models.cache.infinispan.RealmCacheSession.getRealm(RealmCacheSession.java:399) org.keycloak.models.jpa.JpaRealmProvider.getRealms(JpaRealmProvider.java:102) org.keycloak.models.cache.infinispan.RealmCacheSession.getRealms(RealmCacheSession.java:459) org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51) org.keycloak.migration.MigrationModelManager.migrate(MigrationModelManager.java:96) org.keycloak.services.resources.KeycloakApplication.migrateModel(KeycloakApplication.java:245) org.keycloak.services.resources.KeycloakApplication.migrateAndBootstrap(KeycloakApplication.java:186) org.keycloak.services.resources.KeycloakApplication$1.run(KeycloakApplication.java:145) org.keycloak.models.utils.KeycloakModelUtils.runJobInTransaction(KeycloakModelUtils.java:227) org.keycloak.services.resources.KeycloakApplication.<init>(KeycloakApplication.java:136) sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) java.lang.reflect.Constructor.newInstance(Constructor.java:423) org.jboss.resteasy.core.ConstructorInjectorImpl.construct(ConstructorInjectorImpl.java:150) org.jboss.resteasy.spi.ResteasyProviderFactory.createProviderInstance(ResteasyProviderFactory.java:2298) org.jboss.resteasy.spi.ResteasyDeployment.createApplication(ResteasyDeployment.java:340) org.jboss.resteasy.spi.ResteasyDeployment.start(ResteasyDeployment.java:253) org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.init(ServletContainerDispatcher.java:120) org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.init(HttpServletDispatcher.java:36) io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:117) org.wildfly.extension.undertow.security.RunAsLifecycleInterceptor.init(RunAsLifecycleInterceptor.java:78) io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:103) io.undertow.servlet.core.ManagedServlet$DefaultInstanceStrategy.start(ManagedServlet.java:250) io.undertow.servlet.core.ManagedServlet.createServlet(ManagedServlet.java:133) io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:565) io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:536) io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:42) io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43) org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction.lambda$create$0(SecurityContextThreadSetupAction.java:105) org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction$$Lambda$1001/538179304.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown Source) io.undertow.servlet.core.DeploymentManagerImpl.start(DeploymentManagerImpl.java:578) org.wildfly.extension.undertow.deployment.UndertowDeploymentService.startContext(UndertowDeploymentService.java:100) org.wildfly.extension.undertow.deployment.UndertowDeploymentService$1.run(UndertowDeploymentService.java:81) java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:748) org.jboss.threads.JBossThread.run(JBossThread.java:320) 22:16:17,085 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012108: CheckedAction::check - atomic action 0:ffffac110004:-14e6f320:5ba958b2:12 aborting with 1 threads active! 22:16:17,099 WARN [org.hibernate.resource.transaction.backend.jta.internal.synchronization.SynchronizationCallbackCoordinatorTrackingImpl] (Transaction Reaper Worker 0) HHH000 451: Transaction afterCompletion called by a background thread; delaying afterCompletion processing until the original thread can handle it. [status=4] 22:16:17,101 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012121: TransactionReaper::doCancellations worker Thread[Transaction Reaper Worker 0,5,main] succ essfully canceled TX 0:ffffac110004:-14e6f320:5ba958b2:12

I'm having a similar problem with a fresh 4.4.0 KC installation using Mysql, i.e. I also get these transaction timeouts. Unfortunately I did not find a solution, yet. Greetings Henning Am Di., 25. Sep. 2018 um 09:21 Uhr schrieb Marek Posolda < mposolda(a)redhat.com&gt;: > We did not try to test with so many realms yet, so issues are expected > unfortunately :( We plan to improve it, but not sure when... > > To workaround this one, you may need to increase the default JTA > transaction timeout, which is doable by adding/changing some attributes > in the Wildfly standalone.xml configuration file somewhere in the > transaction subsystem. By default it is 10 minutes AFAIR. See Wildfly > docs for more details. > > Marek > > On 25/09/18 01:07, rony joy wrote: > > Dear All, > > > > We have currently using keycloak 3.4.3 version and trying to migrate to > > 4.3.0 but the startup is failing due to the migration issue. We have > around > > 400-500 realms in the database. Please find below exception. From the > log > > it is clear that > > > "org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51)" > > is the one causing the exception (see the code below. Line 51 is in > bold). > > Is this because of large realms? any ideas? > > @Override > > public void migrate(KeycloakSession session) { > > *session.realms().getRealms().stream().forEach(* > > r -> { > > migrateRealm(session, r, false); > > } > > ); > > } > > > > > > 22:16:17,002 WARN [com.arjuna.ats.arjuna] (Transaction Reaper) > > ARJUNA012117: TransactionReaper::check timeout for TX > > 0:ffffac110004:-14e6f320:5ba958b2:12 in state RUN > > 22:16:17,070 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) > > ARJUNA012121: TransactionReaper::doCancellations worker > Thread[Transaction > > Reaper Worker 0,5,main] succ > > essfully canceled TX 0:ffffac110004:-14e6f320:5ba958b2:f > > 22:16:17,073 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) > > ARJUNA012095: Abort of action id 0:ffffac110004:-14e6f320:5ba958b2:12 > > invoked while multiple threads ac > > tive within it. > > 22:16:17,079 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) > > ARJUNA012381: Action id 0:ffffac110004:-14e6f320:5ba958b2:12 completed > with > > multiple threads - thread S > > erverService Thread Pool -- 53 was in progress with > > > org.hibernate.event.internal.DefaultPersistEventListener.entityIsPersistent(DefaultPersistEventListener.java:163) > > > org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:128) > > > org.hibernate.internal.SessionImpl.firePersistOnFlush(SessionImpl.java:805) > > org.hibernate.internal.SessionImpl.persistOnFlush(SessionImpl.java:798) > > > org.hibernate.engine.spi.CascadingActions$8.cascade(CascadingActions.java:340) > > org.hibernate.engine.internal.Cascade.cascadeToOne(Cascade.java:423) > > > org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:348) > > org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) > > > org.hibernate.engine.internal.Cascade.cascadeCollectionElements(Cascade.java:456) > > > org.hibernate.engine.internal.Cascade.cascadeCollection(Cascade.java:388) > > > org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:351) > > org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) > > org.hibernate.engine.internal.Cascade.cascade(Cascade.java:136) > > > org.hibernate.event.internal.AbstractSaveEventListener.cascadeAfterSave(AbstractSaveEventListener.java:445) > > > org.hibernate.event.internal.DefaultPersistEventListener.justCascade(DefaultPersistEventListener.java:172) > > > org.hibernate.event.internal.DefaultPersistEventListener.entityIsPersistent(DefaultPersistEventListener.java:164) > > > org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:128) > > > org.hibernate.internal.SessionImpl.firePersistOnFlush(SessionImpl.java:805) > > org.hibernate.internal.SessionImpl.persistOnFlush(SessionImpl.java:798) > > > org.hibernate.engine.spi.CascadingActions$8.cascade(CascadingActions.java:340) > > org.hibernate.engine.internal.Cascade.cascadeToOne(Cascade.java:423) > > > org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:348) > > org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) > > > org.hibernate.engine.internal.Cascade.cascadeCollectionElements(Cascade.java:456) > > > org.hibernate.engine.internal.Cascade.cascadeCollection(Cascade.java:388) > > > org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:351) > > org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) > > org.hibernate.engine.internal.Cascade.cascade(Cascade.java:136) > > > org.hibernate.event.internal.AbstractFlushingEventListener.cascadeOnFlush(AbstractFlushingEventListener.java:150) > > > org.hibernate.event.internal.AbstractFlushingEventListener.prepareEntityFlushes(AbstractFlushingEventListener.java:141) > > > org.hibernate.event.internal.AbstractFlushingEventListener.flushEverythingToExecutions(AbstractFlushingEventListener.java:74) > > > org.hibernate.event.internal.DefaultAutoFlushEventListener.onAutoFlush(DefaultAutoFlushEventListener.java:44) > > > org.hibernate.internal.SessionImpl.autoFlushIfRequired(SessionImpl.java:1264) > > org.hibernate.internal.SessionImpl.list(SessionImpl.java:1332) > > org.hibernate.internal.QueryImpl.list(QueryImpl.java:87) > > org.hibernate.jpa.internal.QueryImpl.list(QueryImpl.java:606) > > org.hibernate.jpa.internal.QueryImpl.getResultList(QueryImpl.java:483) > > > org.keycloak.models.jpa.ClientAdapter.getClientScopes(ClientAdapter.java:353) > > > org.keycloak.models.cache.infinispan.entities.CachedClient.<init>(CachedClient.java:119) > > > org.keycloak.models.cache.infinispan.RealmCacheSession.cacheClient(RealmCacheSession.java:1069) > > > org.keycloak.models.cache.infinispan.RealmCacheSession.getClientById(RealmCacheSession.java:1029) > > > org.keycloak.models.jpa.RealmAdapter.getMasterAdminClient(RealmAdapter.java:1037) > > > org.keycloak.models.cache.infinispan.entities.CachedRealm.<init>(CachedRealm.java:235) > > > org.keycloak.models.cache.infinispan.RealmCacheSession.getRealm(RealmCacheSession.java:399) > > > org.keycloak.models.jpa.JpaRealmProvider.getRealms(JpaRealmProvider.java:102) > > > org.keycloak.models.cache.infinispan.RealmCacheSession.getRealms(RealmCacheSession.java:459) > > > org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51) > > > org.keycloak.migration.MigrationModelManager.migrate(MigrationModelManager.java:96) > > > org.keycloak.services.resources.KeycloakApplication.migrateModel(KeycloakApplication.java:245) > > > org.keycloak.services.resources.KeycloakApplication.migrateAndBootstrap(KeycloakApplication.java:186) > > > org.keycloak.services.resources.KeycloakApplication$1.run(KeycloakApplication.java:145) > > > org.keycloak.models.utils.KeycloakModelUtils.runJobInTransaction(KeycloakModelUtils.java:227) > > > org.keycloak.services.resources.KeycloakApplication.<init>(KeycloakApplication.java:136) > > sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) > > > sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) > > > sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) > > java.lang.reflect.Constructor.newInstance(Constructor.java:423) > > > org.jboss.resteasy.core.ConstructorInjectorImpl.construct(ConstructorInjectorImpl.java:150) > > > org.jboss.resteasy.spi.ResteasyProviderFactory.createProviderInstance(ResteasyProviderFactory.java:2298) > > > org.jboss.resteasy.spi.ResteasyDeployment.createApplication(ResteasyDeployment.java:340) > > > org.jboss.resteasy.spi.ResteasyDeployment.start(ResteasyDeployment.java:253) > > > org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.init(ServletContainerDispatcher.java:120) > > > org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.init(HttpServletDispatcher.java:36) > > > io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:117) > > > org.wildfly.extension.undertow.security.RunAsLifecycleInterceptor.init(RunAsLifecycleInterceptor.java:78) > > > io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:103) > > > io.undertow.servlet.core.ManagedServlet$DefaultInstanceStrategy.start(ManagedServlet.java:250) > > > io.undertow.servlet.core.ManagedServlet.createServlet(ManagedServlet.java:133) > > > io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:565) > > > io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:536) > > > io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:42) > > > io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43) > > > org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction.lambda$create$0(SecurityContextThreadSetupAction.java:105) > > > org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction$$Lambda$1001/538179304.call(Unknown > > Source) > > > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) > > > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown > > Source) > > > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) > > > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown > > Source) > > > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) > > > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown > > Source) > > > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) > > > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown > > Source) > > > io.undertow.servlet.core.DeploymentManagerImpl.start(DeploymentManagerImpl.java:578) > > > org.wildfly.extension.undertow.deployment.UndertowDeploymentService.startContext(UndertowDeploymentService.java:100) > > > org.wildfly.extension.undertow.deployment.UndertowDeploymentService$1.run(UndertowDeploymentService.java:81) > > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > > java.util.concurrent.FutureTask.run(FutureTask.java:266) > > > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > > > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > > java.lang.Thread.run(Thread.java:748) > > org.jboss.threads.JBossThread.run(JBossThread.java:320) > > > > 22:16:17,085 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) > > ARJUNA012108: CheckedAction::check - atomic action > > 0:ffffac110004:-14e6f320:5ba958b2:12 aborting with 1 > > threads active! > > 22:16:17,099 WARN > > > [org.hibernate.resource.transaction.backend.jta.internal.synchronization.SynchronizationCallbackCoordinatorTrackingImpl] > > (Transaction Reaper Worker 0) HHH000 > > 451: Transaction afterCompletion called by a background thread; delaying > > afterCompletion processing until the original thread can handle it. > > [status=4] > > 22:16:17,101 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) > > ARJUNA012121: TransactionReaper::doCancellations worker > Thread[Transaction > > Reaper Worker 0,5,main] succ > > essfully canceled TX 0:ffffac110004:-14e6f320:5ba958b2:12 > > > > _______________________________________________ > keycloak-user mailing list > keycloak-user(a)lists.jboss.org > https://lists.jboss.org/mailman/listinfo/keycloak-user > -- ----------- Henning Waack | IT Consultant codecentric AG | Hochstraße 11 <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... | <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... Solingen <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... |Deutschland <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... tel: +49 (0)151 108 515 29 www.codecentric.de | blog.codecentric.de | www.meettheexperts.de Sitz der Gesellschaft: Solingen | HRB 25917 | Amtsgericht Wuppertal Vorstand: Michael Hochgürtel . Ulrich Kühn . Rainer Vehns Aufsichtsrat: Patric Fedlmeier (Vorsitzender) . Klaus Jäger . Jürgen Schütz Diese E-Mail einschließlich evtl. beigefügter Dateien enthält vertrauliche und/oder rechtlich geschützte Informationen. Wenn Sie nicht der richtige Adressat sind oder diese E-Mail irrtümlich erhalten haben, informieren Sie bitte sofort den Absender und löschen Sie diese E-Mail und evtl. beigefügter Dateien umgehend. Das unerlaubte Kopieren, Nutzen oder Öffnen evtl. beigefügter Dateien sowie die unbefugte Weitergabe dieser E-Mail ist nicht gestattet.

I have only one tiny Realm, but my problem starts already at installation time, not migration time. Which DB are you using? Am Di., 25. Sep. 2018 um 14:03 Uhr schrieb rony joy <ronyjoy(a)gmail.com&gt;: > How many realms do you have ? > > On Tue, Sep 25, 2018 at 4:50 AM Henning Waack < > henning.waack(a)codecentric.de&gt; wrote: > >> I'm having a similar problem with a fresh 4.4.0 KC installation using >> Mysql, i.e. I also get these transaction timeouts. Unfortunately I did not >> find a solution, yet. >> >> Greetings >> >> Henning >> >> Am Di., 25. Sep. 2018 um 09:21 Uhr schrieb Marek Posolda < >> mposolda(a)redhat.com&gt;: >> >>> We did not try to test with so many realms yet, so issues are expected >>> unfortunately :( We plan to improve it, but not sure when... >>> >>> To workaround this one, you may need to increase the default JTA >>> transaction timeout, which is doable by adding/changing some attributes >>> in the Wildfly standalone.xml configuration file somewhere in the >>> transaction subsystem. By default it is 10 minutes AFAIR. See Wildfly >>> docs for more details. >>> >>> Marek >>> >>> On 25/09/18 01:07, rony joy wrote: >>> > Dear All, >>> > >>> > We have currently using keycloak 3.4.3 version and trying to migrate >>> to >>> > 4.3.0 but the startup is failing due to the migration issue. We have >>> around >>> > 400-500 realms in the database. Please find below exception. From >>> the log >>> > it is clear that >>> > >>> "org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51)" >>> > is the one causing the exception (see the code below. Line 51 is in >>> bold). >>> > Is this because of large realms? any ideas? >>> > @Override >>> > public void migrate(KeycloakSession session) { >>> > *session.realms().getRealms().stream().forEach(* >>> > r -> { >>> > migrateRealm(session, r, false); >>> > } >>> > ); >>> > } >>> > >>> > >>> > 22:16:17,002 WARN [com.arjuna.ats.arjuna] (Transaction Reaper) >>> > ARJUNA012117: TransactionReaper::check timeout for TX >>> > 0:ffffac110004:-14e6f320:5ba958b2:12 in state RUN >>> > 22:16:17,070 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker >>> 0) >>> > ARJUNA012121: TransactionReaper::doCancellations worker >>> Thread[Transaction >>> > Reaper Worker 0,5,main] succ >>> > essfully canceled TX 0:ffffac110004:-14e6f320:5ba958b2:f >>> > 22:16:17,073 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker >>> 0) >>> > ARJUNA012095: Abort of action id 0:ffffac110004:-14e6f320:5ba958b2:12 >>> > invoked while multiple threads ac >>> > tive within it. >>> > 22:16:17,079 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker >>> 0) >>> > ARJUNA012381: Action id 0:ffffac110004:-14e6f320:5ba958b2:12 >>> completed with >>> > multiple threads - thread S >>> > erverService Thread Pool -- 53 was in progress with >>> > >>> org.hibernate.event.internal.DefaultPersistEventListener.entityIsPersistent(DefaultPersistEventListener.java:163) >>> > >>> org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:128) >>> > >>> org.hibernate.internal.SessionImpl.firePersistOnFlush(SessionImpl.java:805) >>> > >>> org.hibernate.internal.SessionImpl.persistOnFlush(SessionImpl.java:798) >>> > >>> org.hibernate.engine.spi.CascadingActions$8.cascade(CascadingActions.java:340) >>> > org.hibernate.engine.internal.Cascade.cascadeToOne(Cascade.java:423) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:348) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeCollectionElements(Cascade.java:456) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeCollection(Cascade.java:388) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:351) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) >>> > org.hibernate.engine.internal.Cascade.cascade(Cascade.java:136) >>> > >>> org.hibernate.event.internal.AbstractSaveEventListener.cascadeAfterSave(AbstractSaveEventListener.java:445) >>> > >>> org.hibernate.event.internal.DefaultPersistEventListener.justCascade(DefaultPersistEventListener.java:172) >>> > >>> org.hibernate.event.internal.DefaultPersistEventListener.entityIsPersistent(DefaultPersistEventListener.java:164) >>> > >>> org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:128) >>> > >>> org.hibernate.internal.SessionImpl.firePersistOnFlush(SessionImpl.java:805) >>> > >>> org.hibernate.internal.SessionImpl.persistOnFlush(SessionImpl.java:798) >>> > >>> org.hibernate.engine.spi.CascadingActions$8.cascade(CascadingActions.java:340) >>> > org.hibernate.engine.internal.Cascade.cascadeToOne(Cascade.java:423) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:348) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeCollectionElements(Cascade.java:456) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeCollection(Cascade.java:388) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:351) >>> > >>> org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) >>> > org.hibernate.engine.internal.Cascade.cascade(Cascade.java:136) >>> > >>> org.hibernate.event.internal.AbstractFlushingEventListener.cascadeOnFlush(AbstractFlushingEventListener.java:150) >>> > >>> org.hibernate.event.internal.AbstractFlushingEventListener.prepareEntityFlushes(AbstractFlushingEventListener.java:141) >>> > >>> org.hibernate.event.internal.AbstractFlushingEventListener.flushEverythingToExecutions(AbstractFlushingEventListener.java:74) >>> > >>> org.hibernate.event.internal.DefaultAutoFlushEventListener.onAutoFlush(DefaultAutoFlushEventListener.java:44) >>> > >>> org.hibernate.internal.SessionImpl.autoFlushIfRequired(SessionImpl.java:1264) >>> > org.hibernate.internal.SessionImpl.list(SessionImpl.java:1332) >>> > org.hibernate.internal.QueryImpl.list(QueryImpl.java:87) >>> > org.hibernate.jpa.internal.QueryImpl.list(QueryImpl.java:606) >>> > org.hibernate.jpa.internal.QueryImpl.getResultList(QueryImpl.java:483) >>> > >>> org.keycloak.models.jpa.ClientAdapter.getClientScopes(ClientAdapter.java:353) >>> > >>> org.keycloak.models.cache.infinispan.entities.CachedClient.<init>(CachedClient.java:119) >>> > >>> org.keycloak.models.cache.infinispan.RealmCacheSession.cacheClient(RealmCacheSession.java:1069) >>> > >>> org.keycloak.models.cache.infinispan.RealmCacheSession.getClientById(RealmCacheSession.java:1029) >>> > >>> org.keycloak.models.jpa.RealmAdapter.getMasterAdminClient(RealmAdapter.java:1037) >>> > >>> org.keycloak.models.cache.infinispan.entities.CachedRealm.<init>(CachedRealm.java:235) >>> > >>> org.keycloak.models.cache.infinispan.RealmCacheSession.getRealm(RealmCacheSession.java:399) >>> > >>> org.keycloak.models.jpa.JpaRealmProvider.getRealms(JpaRealmProvider.java:102) >>> > >>> org.keycloak.models.cache.infinispan.RealmCacheSession.getRealms(RealmCacheSession.java:459) >>> > >>> org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51) >>> > >>> org.keycloak.migration.MigrationModelManager.migrate(MigrationModelManager.java:96) >>> > >>> org.keycloak.services.resources.KeycloakApplication.migrateModel(KeycloakApplication.java:245) >>> > >>> org.keycloak.services.resources.KeycloakApplication.migrateAndBootstrap(KeycloakApplication.java:186) >>> > >>> org.keycloak.services.resources.KeycloakApplication$1.run(KeycloakApplication.java:145) >>> > >>> org.keycloak.models.utils.KeycloakModelUtils.runJobInTransaction(KeycloakModelUtils.java:227) >>> > >>> org.keycloak.services.resources.KeycloakApplication.<init>(KeycloakApplication.java:136) >>> > sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) >>> > >>> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) >>> > >>> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) >>> > java.lang.reflect.Constructor.newInstance(Constructor.java:423) >>> > >>> org.jboss.resteasy.core.ConstructorInjectorImpl.construct(ConstructorInjectorImpl.java:150) >>> > >>> org.jboss.resteasy.spi.ResteasyProviderFactory.createProviderInstance(ResteasyProviderFactory.java:2298) >>> > >>> org.jboss.resteasy.spi.ResteasyDeployment.createApplication(ResteasyDeployment.java:340) >>> > >>> org.jboss.resteasy.spi.ResteasyDeployment.start(ResteasyDeployment.java:253) >>> > >>> org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.init(ServletContainerDispatcher.java:120) >>> > >>> org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.init(HttpServletDispatcher.java:36) >>> > >>> io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:117) >>> > >>> org.wildfly.extension.undertow.security.RunAsLifecycleInterceptor.init(RunAsLifecycleInterceptor.java:78) >>> > >>> io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:103) >>> > >>> io.undertow.servlet.core.ManagedServlet$DefaultInstanceStrategy.start(ManagedServlet.java:250) >>> > >>> io.undertow.servlet.core.ManagedServlet.createServlet(ManagedServlet.java:133) >>> > >>> io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:565) >>> > >>> io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:536) >>> > >>> io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:42) >>> > >>> io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43) >>> > >>> org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction.lambda$create$0(SecurityContextThreadSetupAction.java:105) >>> > >>> org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction$$Lambda$1001/538179304.call(Unknown >>> > Source) >>> > >>> org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) >>> > >>> org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown >>> > Source) >>> > >>> org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) >>> > >>> org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown >>> > Source) >>> > >>> org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) >>> > >>> org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown >>> > Source) >>> > >>> org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) >>> > >>> org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown >>> > Source) >>> > >>> io.undertow.servlet.core.DeploymentManagerImpl.start(DeploymentManagerImpl.java:578) >>> > >>> org.wildfly.extension.undertow.deployment.UndertowDeploymentService.startContext(UndertowDeploymentService.java:100) >>> > >>> org.wildfly.extension.undertow.deployment.UndertowDeploymentService$1.run(UndertowDeploymentService.java:81) >>> > >>> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) >>> > java.util.concurrent.FutureTask.run(FutureTask.java:266) >>> > >>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) >>> > >>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) >>> > java.lang.Thread.run(Thread.java:748) >>> > org.jboss.threads.JBossThread.run(JBossThread.java:320) >>> > >>> > 22:16:17,085 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker >>> 0) >>> > ARJUNA012108: CheckedAction::check - atomic action >>> > 0:ffffac110004:-14e6f320:5ba958b2:12 aborting with 1 >>> > threads active! >>> > 22:16:17,099 WARN >>> > >>> [org.hibernate.resource.transaction.backend.jta.internal.synchronization.SynchronizationCallbackCoordinatorTrackingImpl] >>> > (Transaction Reaper Worker 0) HHH000 >>> > 451: Transaction afterCompletion called by a background thread; >>> delaying >>> > afterCompletion processing until the original thread can handle it. >>> > [status=4] >>> > 22:16:17,101 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker >>> 0) >>> > ARJUNA012121: TransactionReaper::doCancellations worker >>> Thread[Transaction >>> > Reaper Worker 0,5,main] succ >>> > essfully canceled TX 0:ffffac110004:-14e6f320:5ba958b2:12 >>> > >>> >>> _______________________________________________ >>> keycloak-user mailing list >>> keycloak-user(a)lists.jboss.org >>> https://lists.jboss.org/mailman/listinfo/keycloak-user >>> >> >> >> -- >> >> >> ----------- >> >> Henning Waack | IT Consultant >> >> >> codecentric AG | Hochstraße 11 >> <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... >> | >> <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... >> >> <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... >> Solingen >> <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... >> |Deutschland >> <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... >> >> >> tel: +49 (0)151 108 515 29 >> >> www.codecentric.de | blog.codecentric.de | www.meettheexperts.de >> >> Sitz der Gesellschaft: Solingen | HRB 25917 | Amtsgericht Wuppertal >> >> Vorstand: Michael Hochgürtel . Ulrich Kühn . Rainer Vehns >> Aufsichtsrat: Patric Fedlmeier (Vorsitzender) . Klaus Jäger . Jürgen >> Schütz >> >> Diese E-Mail einschließlich evtl. beigefügter Dateien enthält >> vertrauliche und/oder rechtlich geschützte Informationen. Wenn Sie nicht >> der richtige Adressat sind oder diese E-Mail irrtümlich erhalten haben, >> informieren Sie bitte sofort den Absender und löschen Sie diese E-Mail und >> evtl. beigefügter Dateien umgehend. Das unerlaubte Kopieren, Nutzen oder >> Öffnen evtl. beigefügter Dateien sowie die unbefugte Weitergabe dieser >> E-Mail ist nicht gestattet. >> > > > -- > Rony Joy > > -- ----------- Henning Waack | IT Consultant codecentric AG | Hochstraße 11 <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... | <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... Solingen <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... |Deutschland <https://maps.google.com/?q=Hochstra%C3%9Fe+11%C2%A0+%7C+%C2%A0+42697+Soli... tel: +49 (0)151 108 515 29 www.codecentric.de | blog.codecentric.de | www.meettheexperts.de Sitz der Gesellschaft: Solingen | HRB 25917 | Amtsgericht Wuppertal Vorstand: Michael Hochgürtel . Ulrich Kühn . Rainer Vehns Aufsichtsrat: Patric Fedlmeier (Vorsitzender) . Klaus Jäger . Jürgen Schütz Diese E-Mail einschließlich evtl. beigefügter Dateien enthält vertrauliche und/oder rechtlich geschützte Informationen. Wenn Sie nicht der richtige Adressat sind oder diese E-Mail irrtümlich erhalten haben, informieren Sie bitte sofort den Absender und löschen Sie diese E-Mail und evtl. beigefügter Dateien umgehend. Das unerlaubte Kopieren, Nutzen oder Öffnen evtl. beigefügter Dateien sowie die unbefugte Weitergabe dieser E-Mail ist nicht gestattet.

Hi Marek, I forgot to mention that we had increased the transaction timeout to 20 minutes and then to 40 min which is is not helping. It simply takes more than 40 minutes :( to load the realms into cache.  any help appreciated. Attached the keycloak debug logs which shows exactly after 40 minutes transaction timeout occurs. 22:32:02,793 DEBUG [org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider] (ServerService Thread Pool -- 51) Validation passed. Database is up-to-date for changelog META-INF/jpa-changelog-master.xml 22:32:02,804 DEBUG [org.keycloak.connections.jpa.updater.liquibase.conn.DefaultLiquibaseConnectionProvider] (ServerService Thread Pool -- 51) Using changelog file META-INF/sasi dp-config-changelog.xml and changelogTableName DATABASECHANGELOG_SASIDP_CON 22:32:02,830 DEBUG [org.keycloak.connections.jpa.updater.liquibase.conn.DefaultLiquibaseConnectionProvider] (ServerService Thread Pool -- 51) Reading from keycloak3000.DATABASE CHANGELOG_SASIDP_CON 22:32:02,843 DEBUG [org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider] (ServerService Thread Pool -- 51) Validation passed. Database is up-to-date for changelog META-INF/sasidp-config-changelog.xml 22:32:02,844 DEBUG [org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory] (ServerService Thread Pool -- 51) Database is up-to-date 22:32:02name: keycloak-defaultate.jpa.internal.util.LogHelper] (ServerService Thread Pool -- 51) HHH000204: Processing PersistenceUnitInfo [         ...] 22:32:03,000 INFO  [org.hibernate.Version] (ServerService Thread Pool -- 51) HHH000412: Hibernate Core {5.1.10.Final} 22:32:03,004 INFO  [org.hibernate.cfg.Environment] (ServerService Thread Pool -- 51) HHH000206: hibernate.properties not found 22:32:03,007 INFO  [org.hibernate.cfg.Environment] (ServerService Thread Pool -- 51) HHH000021: Bytecode provider name : javassist 22:32:03,056 INFO [org.hibernate.annotations.common.Version] (ServerService Thread Pool -- 51) HCANN000001: Hibernate Commons Annotations {5.0.1.Final} 22:32:03,230 INFO  [org.hibernate.dialect.Dialect] (ServerService Thread Pool -- 51) HHH000400: Using dialect: org.hibernate.dialect.MySQL5Dialect 22:32:03,288 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (Timer-2) new JtaTransactionWrapper 22:32:03,290 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (Timer-2) was existing? false 22:32:03,291 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (Timer-2) new JtaTransactionWrapper 22:32:03,291 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (Timer-2) was existing? true 22:32:03,320 INFO [org.hibernate.envers.boot.internal.EnversServiceImpl] (ServerService Thread Pool -- 51) Envers integration enabled? : true 22:32:04,230 INFO [org.hibernate.validator.internal.util.Version] (ServerService Thread Pool -- 51) HV000001: Hibernate Validator 5.3.5.Final 22:32:05,429 INFO [org.hibernate.hql.internal.QueryTranslatorFactoryInitiator] (ServerService Thread Pool -- 51) HHH000397: Using ASTQueryTranslatorFactory 22:32:07,290 DEBUG [org.keycloak.migration.MigrationModelManager] (ServerService Thread Pool -- 51) Migrating older model to 4.0.0 22:32:07,305 DEBUG [com.gemalto.sas.keycloak.service.crypto.StorageKeyLoader] (Timer-2) Decrypting storage key... 22:32:07,339 DEBUG [com.gemalto.sas.keycloak.service.crypto.StorageKeyLoader] (Timer-2) Storage key decrypted. 22:32:07,340 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (Timer-2) JtaTransactionWrapper  commit 22:32:07,404 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (Timer-2) JtaTransactionWrapper end 22:32:07,404 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (Timer-2) JtaTransactionWrapper resuming suspended 22:32:07,406 DEBUG [org.keycloak.timer.basic.BasicTimerProvider] (Timer-2) Cancelling task 'load-storage-key' 22:32:07,407 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (Timer-2) JtaTransactionWrapper  commit 22:32:07,408 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (Timer-2) JtaTransactionWrapper end 22:32:07,409 DEBUG [org.keycloak.services.scheduled.ScheduledTaskRunner] (Timer-2) Executed scheduled task 22:32:09,853 DEBUG [com.gemalto.sas.keycloak.admin.RoleServiceImpl] (pool-8-thread-1) Creating addIdpCompositeRoles 22:32:22,269 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (pool-8-thread-1) JtaTransactionWrapper  commit 22:32:22,335 DEBUG [org.keycloak.transaction.JtaTransactionWrapper] (pool-8-thread-1) JtaTransactionWrapper end 22:32:22,364 DEBUG [com.gemalto.sas.keycloak.admin.RoleServiceFactory] (pool-8-thread-1) setting up roles completed 23:12:04,365 WARN  [com.arjuna.ats.arjuna] (Transaction Reaper) ARJUNA012117: TransactionReaper::check timeout for TX 0:ffffac110004:268991c5:5ba965d6:10 in state  RUN 23:12:04,425 WARN  [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012121: TransactionReaper::doCancellations worker Thread[Transaction Reaper Worker 0,5,main] succ essfully canceled TX 0:ffffac110004:268991c5:5ba965d6:d 23:12:04,428 WARN  [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012095: Abort of action id 0:ffffac110004:268991c5:5ba965d6:10 invoked while multiple threads act ive within it. 23:12:04,436 WARN  [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012381: Action id 0:ffffac110004:268991c5:5ba965d6:10 completed with multiple threads - thread Se rverService Thread Pool -- 51 was in progress with org.hibernate.type.TypeHelper.findDirty(TypeHelper.java:321) org.hibernate.persister.entity.AbstractEntityPersister.findDirty(AbstractEntityPersister.java:4204) org.hibernate.event.internal.DefaultFlushEntityEventListener.dirtyCheck(DefaultFlushEntityEventListener.java:528) org.hibernate.event.internal.DefaultFlushEntityEventListener.isUpdateNecessary(DefaultFlushEntityEventListener.java:215) org.hibernate.event.internal.DefaultFlushEntityEventListener.onFlushEntity(DefaultFlushEntityEventListener.java:142) org.hibernate.event.internal.AbstractFlushingEventListener.flushEntities(AbstractFlushingEventListener.java:216) org.hibernate.event.internal.AbstractFlushingEventListener.flushEverythingToExecutions(AbstractFlushingEventListener.java:85) org.hibernate.event.internal.DefaultAutoFlushEventListener.onAutoFlush(DefaultAutoFlushEventListener.java:44) org.hibernate.internal.SessionImpl.autoFlushIfRequired(SessionImpl.java:1264) org.hibernate.internal.SessionImpl.list(SessionImpl.java:1332) org.hibernate.internal.QueryImpl.list(QueryImpl.java:87) org.hibernate.jpa.internal.QueryImpl.list(QueryImpl.java:606) org.hibernate.jpa.internal.QueryImpl.getResultList(QueryImpl.java:483) org.keycloak.models.jpa.RealmAdapter.getDefaultClientScopes(RealmAdapter.java:1893) org.keycloak.models.cache.infinispan.entities.CachedRealm.cacheClientScopes(CachedRealm.java:294) org.keycloak.models.cache.infinispan.entities.CachedRealm.<init>(CachedRealm.java:238) org.keycloak.models.cache.infinispan.RealmCacheSession.getRealm(RealmCacheSession.java:399) org.keycloak.models.jpa.JpaRealmProvider.getRealms(JpaRealmProvider.java:102) org.keycloak.models.cache.infinispan.RealmCacheSession.getRealms(RealmCacheSession.java:459) org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51) org.keycloak.migration.MigrationModelManager.migrate(MigrationModelManager.java:96) org.keycloak.services.resources.KeycloakApplication.migrateModel(KeycloakApplication.java:245) org.keycloak.services.resources.KeycloakApplication.migrateAndBootstrap(KeycloakApplication.java:186) org.keycloak.services.resources.KeycloakApplication$1.run(KeycloakApplication.java:145) org.keycloak.models.utils.KeycloakModelUtils.runJobInTransaction(KeycloakModelUtils.java:227) org.keycloak.services.resources.KeycloakApplication.<init>(KeycloakApplication.java:136) sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) java.lang.reflect.Constructor.newInstance(Constructor.java:423) org.jboss.resteasy.core.ConstructorInjectorImpl.construct(ConstructorInjectorImpl.java:150) org.jboss.resteasy.spi.ResteasyProviderFactory.createProviderInstance(ResteasyProviderFactory.java:2298) org.jboss.resteasy.spi.ResteasyDeployment.createApplication(ResteasyDeployment.java:340) org.jboss.resteasy.spi.ResteasyDeployment.start(ResteasyDeployment.java:253) org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.init(ServletContainerDispatcher.java:120) org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.init(HttpServletDispatcher.java:36) io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:117) org.wildfly.extension.undertow.security.RunAsLifecycleInterceptor.init(RunAsLifecycleInterceptor.java:78) io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:103) io.undertow.servlet.core.ManagedServlet$DefaultInstanceStrategy.start(ManagedServlet.java:250) io.undertow.servlet.core.ManagedServlet.createServlet(ManagedServlet.java:133) io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:565) io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:536) io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:42) io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43) org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction.lambda$create$0(SecurityContextThreadSetupAction.java:105) org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction$$Lambda$1001/680156514.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/248363466.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/248363466.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/248363466.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/248363466.call(Unknown Source) io.undertow.servlet.core.DeploymentManagerImpl.start(DeploymentManagerImpl.java:578) org.wildfly.extension.undertow.deployment.UndertowDeploymentService.startContext(UndertowDeploymentService.java:100) org.wildfly.extension.undertow.deployment.UndertowDeploymentService$1.run(UndertowDeploymentService.java:81) java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:748) org.jboss.threads.JBossThread.run(JBossThread.java:320) On Tue, Sep 25, 2018 at 2:20 AM Marek Posolda <mposolda(a)redhat.com <mailto:mposolda@redhat.com>> wrote: We did not try to test with so many realms yet, so issues are expected unfortunately :( We plan to improve it, but not sure when... To workaround this one, you may need to increase the default JTA transaction timeout, which is doable by adding/changing some attributes in the Wildfly standalone.xml configuration file somewhere in the transaction subsystem. By default it is 10 minutes AFAIR. See Wildfly docs for more details. Marek On 25/09/18 01:07, rony joy wrote: > Dear All, > > We have currently using keycloak 3.4.3 version and trying to migrate to > 4.3.0 but the startup is failing due to the migration issue. We have around > 400-500 realms in the database.  Please find below exception. From the log > it is clear that > "org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51)" > is the one causing the exception (see the code below. Line 51 is in bold). > Is this because of large realms? any ideas? >   @Override >      public void migrate(KeycloakSession session) { >          *session.realms().getRealms().stream().forEach(* >                  r -> { >                      migrateRealm(session, r, false); >                  } >          ); >      } > > > 22:16:17,002 WARN  [com.arjuna.ats.arjuna] (Transaction Reaper) > ARJUNA012117: TransactionReaper::check timeout for TX > 0:ffffac110004:-14e6f320:5ba958b2:12 in state  RUN > 22:16:17,070 WARN  [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) > ARJUNA012121: TransactionReaper::doCancellations worker Thread[Transaction > Reaper Worker 0,5,main] succ > essfully canceled TX 0:ffffac110004:-14e6f320:5ba958b2:f > 22:16:17,073 WARN  [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) > ARJUNA012095: Abort of action id 0:ffffac110004:-14e6f320:5ba958b2:12 > invoked while multiple threads ac > tive within it. > 22:16:17,079 WARN  [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) > ARJUNA012381: Action id 0:ffffac110004:-14e6f320:5ba958b2:12 completed with > multiple threads - thread S > erverService Thread Pool -- 53 was in progress with > org.hibernate.event.internal.DefaultPersistEventListener.entityIsPersistent(DefaultPersistEventListener.java:163) > org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:128) > org.hibernate.internal.SessionImpl.firePersistOnFlush(SessionImpl.java:805) > org.hibernate.internal.SessionImpl.persistOnFlush(SessionImpl.java:798) > org.hibernate.engine.spi.CascadingActions$8.cascade(CascadingActions.java:340) > org.hibernate.engine.internal.Cascade.cascadeToOne(Cascade.java:423) > org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:348) > org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) > org.hibernate.engine.internal.Cascade.cascadeCollectionElements(Cascade.java:456) > org.hibernate.engine.internal.Cascade.cascadeCollection(Cascade.java:388) > org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:351) > org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) > org.hibernate.engine.internal.Cascade.cascade(Cascade.java:136) > org.hibernate.event.internal.AbstractSaveEventListener.cascadeAfterSave(AbstractSaveEventListener.java:445) > org.hibernate.event.internal.DefaultPersistEventListener.justCascade(DefaultPersistEventListener.java:172) > org.hibernate.event.internal.DefaultPersistEventListener.entityIsPersistent(DefaultPersistEventListener.java:164) > org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:128) > org.hibernate.internal.SessionImpl.firePersistOnFlush(SessionImpl.java:805) > org.hibernate.internal.SessionImpl.persistOnFlush(SessionImpl.java:798) > org.hibernate.engine.spi.CascadingActions$8.cascade(CascadingActions.java:340) > org.hibernate.engine.internal.Cascade.cascadeToOne(Cascade.java:423) > org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:348) > org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) > org.hibernate.engine.internal.Cascade.cascadeCollectionElements(Cascade.java:456) > org.hibernate.engine.internal.Cascade.cascadeCollection(Cascade.java:388) > org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:351) > org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) > org.hibernate.engine.internal.Cascade.cascade(Cascade.java:136) > org.hibernate.event.internal.AbstractFlushingEventListener.cascadeOnFlush(AbstractFlushingEventListener.java:150) > org.hibernate.event.internal.AbstractFlushingEventListener.prepareEntityFlushes(AbstractFlushingEventListener.java:141) > org.hibernate.event.internal.AbstractFlushingEventListener.flushEverythingToExecutions(AbstractFlushingEventListener.java:74) > org.hibernate.event.internal.DefaultAutoFlushEventListener.onAutoFlush(DefaultAutoFlushEventListener.java:44) > org.hibernate.internal.SessionImpl.autoFlushIfRequired(SessionImpl.java:1264) > org.hibernate.internal.SessionImpl.list(SessionImpl.java:1332) > org.hibernate.internal.QueryImpl.list(QueryImpl.java:87) > org.hibernate.jpa.internal.QueryImpl.list(QueryImpl.java:606) > org.hibernate.jpa.internal.QueryImpl.getResultList(QueryImpl.java:483) > org.keycloak.models.jpa.ClientAdapter.getClientScopes(ClientAdapter.java:353) > org.keycloak.models.cache.infinispan.entities.CachedClient.<init>(CachedClient.java:119) > org.keycloak.models.cache.infinispan.RealmCacheSession.cacheClient(RealmCacheSession.java:1069) > org.keycloak.models.cache.infinispan.RealmCacheSession.getClientById(RealmCacheSession.java:1029) > org.keycloak.models.jpa.RealmAdapter.getMasterAdminClient(RealmAdapter.java:1037) > org.keycloak.models.cache.infinispan.entities.CachedRealm.<init>(CachedRealm.java:235) > org.keycloak.models.cache.infinispan.RealmCacheSession.getRealm(RealmCacheSession.java:399) > org.keycloak.models.jpa.JpaRealmProvider.getRealms(JpaRealmProvider.java:102) > org.keycloak.models.cache.infinispan.RealmCacheSession.getRealms(RealmCacheSession.java:459) > org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51) > org.keycloak.migration.MigrationModelManager.migrate(MigrationModelManager.java:96) > org.keycloak.services.resources.KeycloakApplication.migrateModel(KeycloakApplication.java:245) > org.keycloak.services.resources.KeycloakApplication.migrateAndBootstrap(KeycloakApplication.java:186) > org.keycloak.services.resources.KeycloakApplication$1.run(KeycloakApplication.java:145) > org.keycloak.models.utils.KeycloakModelUtils.runJobInTransaction(KeycloakModelUtils.java:227) > org.keycloak.services.resources.KeycloakApplication.<init>(KeycloakApplication.java:136) > sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) > sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) > sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) > java.lang.reflect.Constructor.newInstance(Constructor.java:423) > org.jboss.resteasy.core.ConstructorInjectorImpl.construct(ConstructorInjectorImpl.java:150) > org.jboss.resteasy.spi.ResteasyProviderFactory.createProviderInstance(ResteasyProviderFactory.java:2298) > org.jboss.resteasy.spi.ResteasyDeployment.createApplication(ResteasyDeployment.java:340) > org.jboss.resteasy.spi.ResteasyDeployment.start(ResteasyDeployment.java:253) > org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.init(ServletContainerDispatcher.java:120) > org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.init(HttpServletDispatcher.java:36) > io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:117) > org.wildfly.extension.undertow.security.RunAsLifecycleInterceptor.init(RunAsLifecycleInterceptor.java:78) > io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:103) > io.undertow.servlet.core.ManagedServlet$DefaultInstanceStrategy.start(ManagedServlet.java:250) > io.undertow.servlet.core.ManagedServlet.createServlet(ManagedServlet.java:133) > io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:565) > io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:536) > io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:42) > io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43) > org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction.lambda$create$0(SecurityContextThreadSetupAction.java:105) > org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction$$Lambda$1001/538179304.call(Unknown > Source) > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown > Source) > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown > Source) > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown > Source) > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) > org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown > Source) > io.undertow.servlet.core.DeploymentManagerImpl.start(DeploymentManagerImpl.java:578) > org.wildfly.extension.undertow.deployment.UndertowDeploymentService.startContext(UndertowDeploymentService.java:100) > org.wildfly.extension.undertow.deployment.UndertowDeploymentService$1.run(UndertowDeploymentService.java:81) > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > java.util.concurrent.FutureTask.run(FutureTask.java:266) > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > java.lang.Thread.run(Thread.java:748) > org.jboss.threads.JBossThread.run(JBossThread.java:320) > > 22:16:17,085 WARN  [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) > ARJUNA012108: CheckedAction::check - atomic action > 0:ffffac110004:-14e6f320:5ba958b2:12 aborting with 1 >   threads active! > 22:16:17,099 WARN > [org.hibernate.resource.transaction.backend.jta.internal.synchronization.SynchronizationCallbackCoordinatorTrackingImpl] > (Transaction Reaper Worker 0) HHH000 > 451: Transaction afterCompletion called by a background thread; delaying > afterCompletion processing until the original thread can handle it. > [status=4] > 22:16:17,101 WARN  [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) > ARJUNA012121: TransactionReaper::doCancellations worker Thread[Transaction > Reaper Worker 0,5,main] succ > essfully canceled TX 0:ffffac110004:-14e6f320:5ba958b2:12 > -- Rony Joy