[keycloak-user] Keycloak 3.4.3 to 4.X.X Migration Fails - we have 400-500 realms

Dear All, We have currently using keycloak 3.4.3 version and trying to migrate to 4.3.0 but the startup is failing due to the migration issue. We have around 400-500 realms in the database. Please find below exception. From the log it is clear that "org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51)" is the one causing the exception (see the code below. Line 51 is in bold). Is this because of large realms? any ideas? @Override public void migrate(KeycloakSession session) { *session.realms().getRealms().stream().forEach(* r -> { migrateRealm(session, r, false); } ); } 22:16:17,002 WARN [com.arjuna.ats.arjuna] (Transaction Reaper) ARJUNA012117: TransactionReaper::check timeout for TX 0:ffffac110004:-14e6f320:5ba958b2:12 in state RUN 22:16:17,070 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012121: TransactionReaper::doCancellations worker Thread[Transaction Reaper Worker 0,5,main] succ essfully canceled TX 0:ffffac110004:-14e6f320:5ba958b2:f 22:16:17,073 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012095: Abort of action id 0:ffffac110004:-14e6f320:5ba958b2:12 invoked while multiple threads ac tive within it. 22:16:17,079 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012381: Action id 0:ffffac110004:-14e6f320:5ba958b2:12 completed with multiple threads - thread S erverService Thread Pool -- 53 was in progress with org.hibernate.event.internal.DefaultPersistEventListener.entityIsPersistent(DefaultPersistEventListener.java:163) org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:128) org.hibernate.internal.SessionImpl.firePersistOnFlush(SessionImpl.java:805) org.hibernate.internal.SessionImpl.persistOnFlush(SessionImpl.java:798) org.hibernate.engine.spi.CascadingActions$8.cascade(CascadingActions.java:340) org.hibernate.engine.internal.Cascade.cascadeToOne(Cascade.java:423) org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:348) org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) org.hibernate.engine.internal.Cascade.cascadeCollectionElements(Cascade.java:456) org.hibernate.engine.internal.Cascade.cascadeCollection(Cascade.java:388) org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:351) org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) org.hibernate.engine.internal.Cascade.cascade(Cascade.java:136) org.hibernate.event.internal.AbstractSaveEventListener.cascadeAfterSave(AbstractSaveEventListener.java:445) org.hibernate.event.internal.DefaultPersistEventListener.justCascade(DefaultPersistEventListener.java:172) org.hibernate.event.internal.DefaultPersistEventListener.entityIsPersistent(DefaultPersistEventListener.java:164) org.hibernate.event.internal.DefaultPersistEventListener.onPersist(DefaultPersistEventListener.java:128) org.hibernate.internal.SessionImpl.firePersistOnFlush(SessionImpl.java:805) org.hibernate.internal.SessionImpl.persistOnFlush(SessionImpl.java:798) org.hibernate.engine.spi.CascadingActions$8.cascade(CascadingActions.java:340) org.hibernate.engine.internal.Cascade.cascadeToOne(Cascade.java:423) org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:348) org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) org.hibernate.engine.internal.Cascade.cascadeCollectionElements(Cascade.java:456) org.hibernate.engine.internal.Cascade.cascadeCollection(Cascade.java:388) org.hibernate.engine.internal.Cascade.cascadeAssociation(Cascade.java:351) org.hibernate.engine.internal.Cascade.cascadeProperty(Cascade.java:187) org.hibernate.engine.internal.Cascade.cascade(Cascade.java:136) org.hibernate.event.internal.AbstractFlushingEventListener.cascadeOnFlush(AbstractFlushingEventListener.java:150) org.hibernate.event.internal.AbstractFlushingEventListener.prepareEntityFlushes(AbstractFlushingEventListener.java:141) org.hibernate.event.internal.AbstractFlushingEventListener.flushEverythingToExecutions(AbstractFlushingEventListener.java:74) org.hibernate.event.internal.DefaultAutoFlushEventListener.onAutoFlush(DefaultAutoFlushEventListener.java:44) org.hibernate.internal.SessionImpl.autoFlushIfRequired(SessionImpl.java:1264) org.hibernate.internal.SessionImpl.list(SessionImpl.java:1332) org.hibernate.internal.QueryImpl.list(QueryImpl.java:87) org.hibernate.jpa.internal.QueryImpl.list(QueryImpl.java:606) org.hibernate.jpa.internal.QueryImpl.getResultList(QueryImpl.java:483) org.keycloak.models.jpa.ClientAdapter.getClientScopes(ClientAdapter.java:353) org.keycloak.models.cache.infinispan.entities.CachedClient.<init>(CachedClient.java:119) org.keycloak.models.cache.infinispan.RealmCacheSession.cacheClient(RealmCacheSession.java:1069) org.keycloak.models.cache.infinispan.RealmCacheSession.getClientById(RealmCacheSession.java:1029) org.keycloak.models.jpa.RealmAdapter.getMasterAdminClient(RealmAdapter.java:1037) org.keycloak.models.cache.infinispan.entities.CachedRealm.<init>(CachedRealm.java:235) org.keycloak.models.cache.infinispan.RealmCacheSession.getRealm(RealmCacheSession.java:399) org.keycloak.models.jpa.JpaRealmProvider.getRealms(JpaRealmProvider.java:102) org.keycloak.models.cache.infinispan.RealmCacheSession.getRealms(RealmCacheSession.java:459) org.keycloak.migration.migrators.MigrateTo4_0_0.migrate(MigrateTo4_0_0.java:51) org.keycloak.migration.MigrationModelManager.migrate(MigrationModelManager.java:96) org.keycloak.services.resources.KeycloakApplication.migrateModel(KeycloakApplication.java:245) org.keycloak.services.resources.KeycloakApplication.migrateAndBootstrap(KeycloakApplication.java:186) org.keycloak.services.resources.KeycloakApplication$1.run(KeycloakApplication.java:145) org.keycloak.models.utils.KeycloakModelUtils.runJobInTransaction(KeycloakModelUtils.java:227) org.keycloak.services.resources.KeycloakApplication.<init>(KeycloakApplication.java:136) sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) java.lang.reflect.Constructor.newInstance(Constructor.java:423) org.jboss.resteasy.core.ConstructorInjectorImpl.construct(ConstructorInjectorImpl.java:150) org.jboss.resteasy.spi.ResteasyProviderFactory.createProviderInstance(ResteasyProviderFactory.java:2298) org.jboss.resteasy.spi.ResteasyDeployment.createApplication(ResteasyDeployment.java:340) org.jboss.resteasy.spi.ResteasyDeployment.start(ResteasyDeployment.java:253) org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.init(ServletContainerDispatcher.java:120) org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.init(HttpServletDispatcher.java:36) io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:117) org.wildfly.extension.undertow.security.RunAsLifecycleInterceptor.init(RunAsLifecycleInterceptor.java:78) io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:103) io.undertow.servlet.core.ManagedServlet$DefaultInstanceStrategy.start(ManagedServlet.java:250) io.undertow.servlet.core.ManagedServlet.createServlet(ManagedServlet.java:133) io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:565) io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:536) io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:42) io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43) org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction.lambda$create$0(SecurityContextThreadSetupAction.java:105) org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction$$Lambda$1001/538179304.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown Source) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1508) org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction$$Lambda$1002/1005208678.call(Unknown Source) io.undertow.servlet.core.DeploymentManagerImpl.start(DeploymentManagerImpl.java:578) org.wildfly.extension.undertow.deployment.UndertowDeploymentService.startContext(UndertowDeploymentService.java:100) org.wildfly.extension.undertow.deployment.UndertowDeploymentService$1.run(UndertowDeploymentService.java:81) java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:748) org.jboss.threads.JBossThread.run(JBossThread.java:320) 22:16:17,085 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012108: CheckedAction::check - atomic action 0:ffffac110004:-14e6f320:5ba958b2:12 aborting with 1 threads active! 22:16:17,099 WARN [org.hibernate.resource.transaction.backend.jta.internal.synchronization.SynchronizationCallbackCoordinatorTrackingImpl] (Transaction Reaper Worker 0) HHH000 451: Transaction afterCompletion called by a background thread; delaying afterCompletion processing until the original thread can handle it. [status=4] 22:16:17,101 WARN [com.arjuna.ats.arjuna] (Transaction Reaper Worker 0) ARJUNA012121: TransactionReaper::doCancellations worker Thread[Transaction Reaper Worker 0,5,main] succ essfully canceled TX 0:ffffac110004:-14e6f320:5ba958b2:12 -- Rony Joy