Yes I can confirm the filter is correctly added. I also have all the other required settings configured (hopefully correctly). It works reliably when I target the apache reverse proxy but only falls appart when I load balance through another "hop". I switched on debug on keycloak and can see a bit more details of the failure (logs below). I wonder what "AuthenticationFlowException: Client was not identified by any client authenticator" means. 


<server name="default-server">
    <ajp-listener name="ajpListener" scheme="https" socket-binding="ajp" redirect-socket="proxy-https" enabled="true" />
    <http-listener name="default" socket-binding="http" proxy-address-forwarding="true" redirect-socket="proxy-https" />
    <host name="default-host" alias="localhost">
        <location name="/" handler="welcome-content"/>
        <filter-ref name="server-header"/>
        <filter-ref name="x-powered-by-header"/>
        <filter-ref name="proxy-peer"/>
    </host>
</server>

<filters>
  <response-header name="server-header" header-name="Server" header-value="WildFly/10"/>
  <response-header name="x-powered-by-header" header-name="X-Powered-By" header-value="Undertow/1"/>
  <filter name="proxy-peer" class-name="io.undertow.server.handlers.ProxyPeerAddressHandler" module="io.undertow.core" />
</filters>


<socket-binding-group name="standard-sockets" default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">
  ...
  <socket-binding name="proxy-https" port="443"/>
  <socket-binding name="ajp" port="${jboss.ajp.port:8009}"/>
  <socket-binding name="http" port="${jboss.http.port:8080}"/>
  <socket-binding name="https" port="${jboss.https.port:8443}"/>
  ...
</socket-binding-group>



2016-05-24 09:59:26,176 DEBUG [org.keycloak.services] (default task-11) AUTHENTICATE CLIENT
2016-05-24 09:59:26,176 DEBUG [org.keycloak.services] (default task-11) client authenticator: client-secret
2016-05-24 09:59:26,179 DEBUG [org.keycloak.services] (default task-11) client authenticator: client-jwt
2016-05-24 09:59:26,180 DEBUG [org.keycloak.services] (default task-11) KC-SERVICES0014: Failed client authentication: org.keycloak.authentication.AuthenticationFlowException: Client was not identified by any client authenticator
at org.keycloak.authentication.ClientAuthenticationFlow.processFlow(ClientAuthenticationFlow.java:101)
at org.keycloak.authentication.AuthenticationProcessor.authenticateClient(AuthenticationProcessor.java:673)
at org.keycloak.protocol.oidc.utils.AuthorizeClientUtil.authorizeClient(AuthorizeClientUtil.java:42)
at org.keycloak.protocol.oidc.endpoints.TokenEndpoint.checkClient(TokenEndpoint.java:163)
at org.keycloak.protocol.oidc.endpoints.TokenEndpoint.build(TokenEndpoint.java:117)
at sun.reflect.GeneratedMethodAccessor93.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:139)
at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTarget(ResourceMethodInvoker.java:295)
at org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:249)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:138)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:107)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:133)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:101)
at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:395)
at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:202)
at org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.service(ServletContainerDispatcher.java:221)
at org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:56)
at org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:51)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at io.undertow.servlet.handlers.ServletHandler.handleRequest(ServletHandler.java:85)
at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:129)
at org.keycloak.services.filters.KeycloakSessionServletFilter.doFilter(KeycloakSessionServletFilter.java:88)
at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:60)
at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131)
at io.undertow.servlet.handlers.FilterHandler.handleRequest(FilterHandler.java:84)
at io.undertow.servlet.handlers.security.ServletSecurityRoleHandler.handleRequest(ServletSecurityRoleHandler.java:62)
at io.undertow.servlet.handlers.ServletDispatchingHandler.handleRequest(ServletDispatchingHandler.java:36)
at org.wildfly.extension.undertow.security.SecurityContextAssociationHandler.handleRequest(SecurityContextAssociationHandler.java:78)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.servlet.handlers.security.SSLInformationAssociationHandler.handleRequest(SSLInformationAssociationHandler.java:131)
at io.undertow.servlet.handlers.security.ServletAuthenticationCallHandler.handleRequest(ServletAuthenticationCallHandler.java:57)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.security.handlers.AbstractConfidentialityHandler.handleRequest(AbstractConfidentialityHandler.java:46)
at io.undertow.servlet.handlers.security.ServletConfidentialityConstraintHandler.handleRequest(ServletConfidentialityConstraintHandler.java:64)
at io.undertow.security.handlers.AuthenticationMechanismsHandler.handleRequest(AuthenticationMechanismsHandler.java:60)
at io.undertow.servlet.handlers.security.CachedAuthenticatedSessionHandler.handleRequest(CachedAuthenticatedSessionHandler.java:77)
at io.undertow.security.handlers.NotificationReceiverHandler.handleRequest(NotificationReceiverHandler.java:50)
at io.undertow.security.handlers.AbstractSecurityContextAssociationHandler.handleRequest(AbstractSecurityContextAssociationHandler.java:43)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at org.wildfly.extension.undertow.security.jacc.JACCContextIdHandler.handleRequest(JACCContextIdHandler.java:61)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:284)
at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:263)
at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:81)
at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:174)
at io.undertow.server.Connectors.executeRootHandler(Connectors.java:202)
at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:793)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

2016-05-24 09:59:26,180 WARN  [org.keycloak.events] (default task-11) type=CODE_TO_TOKEN_ERROR, realmId=master, clientId=admin, userId=null, ipAddress=192.168.33.1, error=invalid_client_credentials, grant_type=authorization_code
2016-05-24 09:59:26,863 DEBUG [org.keycloak.services] (default task-8) replacing relative valid redirect with: https://login.vagrant.v8/auth/admin/master/console/*
2016-05-24 09:59:26,867 DEBUG [org.keycloak.services] (default task-8) AUTHENTICATE
2016-05-24 09:59:26,867 DEBUG [org.keycloak.services] (default task-8) AUTHENTICATE ONLY
2016-05-24 09:59:26,867 DEBUG [org.keycloak.services] (default task-8) processFlow
2016-05-24 09:59:26,868 DEBUG [org.keycloak.services] (default task-8) check execution: auth-cookie requirement: ALTERNATIVE
2016-05-24 09:59:26,868 DEBUG [org.keycloak.services] (default task-8) authenticator: auth-cookie
2016-05-24 09:59:26,868 DEBUG [org.keycloak.services] (default task-8) invoke authenticator.authenticate
2016-05-24 09:59:26,869 DEBUG [org.keycloak.services] (default task-8) token active - active: true, issued-at: 1,464,083,965, not-before: 0
2016-05-24 09:59:26,869 DEBUG [org.keycloak.services] (default task-8) authenticator SUCCESS: auth-cookie
2016-05-24 09:59:26,873 DEBUG [org.keycloak.services] (default task-8) check execution: auth-spnego requirement: DISABLED
2016-05-24 09:59:26,873 DEBUG [org.keycloak.services] (default task-8) execution is processed
2016-05-24 09:59:26,873 DEBUG [org.keycloak.services] (default task-8) check execution: null requirement: ALTERNATIVE
2016-05-24 09:59:26,873 DEBUG [org.keycloak.services] (default task-8) Skip alternative execution
2016-05-24 09:59:26,874 DEBUG [org.keycloak.events] (default task-8) type=LOGIN, realmId=master, clientId=security-admin-console, userId=cf248811-6d82-47e7-bf9a-7b197fb988d0, ipAddress=192.168.33.1, auth_method=openid-connect, auth_type=code, response_type=code, redirect_uri=https://login.vagrant.v8/auth/admin/master/console/, consent=no_consent_required, code_id=8215bbb3-a592-47e0-9c28-c274bccc61b5, response_mode=fragment, username=admin
2016-05-24 09:59:26,891 DEBUG [org.keycloak.services] (default task-8) Create login cookie - name: KEYCLOAK_IDENTITY, path: /auth/realms/master, max-age: -1
2016-05-24 09:59:26,892 DEBUG [org.keycloak.services] (default task-8) redirectAccessCode: state: ae0d05d7-95d9-40a0-9a75-7e4e4fb7830e
2016-05-24 09:59:27,472 DEBUG [org.keycloak.services] (default task-11) AUTHENTICATE CLIENT
2016-05-24 09:59:27,472 DEBUG [org.keycloak.services] (default task-11) client authenticator: client-secret
2016-05-24 09:59:27,474 DEBUG [org.keycloak.services] (default task-11) client authenticator: client-jwt
2016-05-24 09:59:27,474 DEBUG [org.keycloak.services] (default task-11) KC-SERVICES0014: Failed client authentication: org.keycloak.authentication.AuthenticationFlowException: Client was not identified by any client authenticator
at org.keycloak.authentication.ClientAuthenticationFlow.processFlow(ClientAuthenticationFlow.java:101)
at org.keycloak.authentication.AuthenticationProcessor.authenticateClient(AuthenticationProcessor.java:673)
at org.keycloak.protocol.oidc.utils.AuthorizeClientUtil.authorizeClient(AuthorizeClientUtil.java:42)
at org.keycloak.protocol.oidc.endpoints.TokenEndpoint.checkClient(TokenEndpoint.java:163)
at org.keycloak.protocol.oidc.endpoints.TokenEndpoint.build(TokenEndpoint.java:117)
at sun.reflect.GeneratedMethodAccessor93.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:139)
at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTarget(ResourceMethodInvoker.java:295)
at org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:249)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:138)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:107)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:133)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:101)
at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:395)
at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:202)
at org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.service(ServletContainerDispatcher.java:221)
at org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:56)
at org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:51)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at io.undertow.servlet.handlers.ServletHandler.handleRequest(ServletHandler.java:85)
at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:129)
at org.keycloak.services.filters.KeycloakSessionServletFilter.doFilter(KeycloakSessionServletFilter.java:88)
at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:60)
at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131)
at io.undertow.servlet.handlers.FilterHandler.handleRequest(FilterHandler.java:84)
at io.undertow.servlet.handlers.security.ServletSecurityRoleHandler.handleRequest(ServletSecurityRoleHandler.java:62)
at io.undertow.servlet.handlers.ServletDispatchingHandler.handleRequest(ServletDispatchingHandler.java:36)
at org.wildfly.extension.undertow.security.SecurityContextAssociationHandler.handleRequest(SecurityContextAssociationHandler.java:78)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.servlet.handlers.security.SSLInformationAssociationHandler.handleRequest(SSLInformationAssociationHandler.java:131)
at io.undertow.servlet.handlers.security.ServletAuthenticationCallHandler.handleRequest(ServletAuthenticationCallHandler.java:57)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.security.handlers.AbstractConfidentialityHandler.handleRequest(AbstractConfidentialityHandler.java:46)
at io.undertow.servlet.handlers.security.ServletConfidentialityConstraintHandler.handleRequest(ServletConfidentialityConstraintHandler.java:64)
at io.undertow.security.handlers.AuthenticationMechanismsHandler.handleRequest(AuthenticationMechanismsHandler.java:60)
at io.undertow.servlet.handlers.security.CachedAuthenticatedSessionHandler.handleRequest(CachedAuthenticatedSessionHandler.java:77)
at io.undertow.security.handlers.NotificationReceiverHandler.handleRequest(NotificationReceiverHandler.java:50)
at io.undertow.security.handlers.AbstractSecurityContextAssociationHandler.handleRequest(AbstractSecurityContextAssociationHandler.java:43)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at org.wildfly.extension.undertow.security.jacc.JACCContextIdHandler.handleRequest(JACCContextIdHandler.java:61)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:284)
at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:263)
at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:81)
at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:174)
at io.undertow.server.Connectors.executeRootHandler(Connectors.java:202)
at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:793)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

2016-05-24 09:59:27,478 WARN  [org.keycloak.events] (default task-11) type=CODE_TO_TOKEN_ERROR, realmId=master, clientId=admin, userId=null, ipAddress=192.168.33.1, error=invalid_client_credentials, grant_type=authorization_code
2016-05-24 09:59:28,209 DEBUG [org.keycloak.services] (default task-8) replacing relative valid redirect with: https://login.vagrant.v8/auth/admin/master/console/*
2016-05-24 09:59:28,211 DEBUG [org.keycloak.services] (default task-8) AUTHENTICATE
2016-05-24 09:59:28,218 DEBUG [org.keycloak.services] (default task-8) AUTHENTICATE ONLY
2016-05-24 09:59:28,218 DEBUG [org.keycloak.services] (default task-8) processFlow
2016-05-24 09:59:28,219 DEBUG [org.keycloak.services] (default task-8) check execution: auth-cookie requirement: ALTERNATIVE
2016-05-24 09:59:28,219 DEBUG [org.keycloak.services] (default task-8) authenticator: auth-cookie
2016-05-24 09:59:28,219 DEBUG [org.keycloak.services] (default task-8) invoke authenticator.authenticate
2016-05-24 09:59:28,220 DEBUG [org.keycloak.services] (default task-8) token active - active: true, issued-at: 1,464,083,966, not-before: 0
2016-05-24 09:59:28,222 DEBUG [org.keycloak.services] (default task-8) authenticator SUCCESS: auth-cookie
2016-05-24 09:59:28,222 DEBUG [org.keycloak.services] (default task-8) check execution: auth-spnego requirement: DISABLED
2016-05-24 09:59:28,222 DEBUG [org.keycloak.services] (default task-8) execution is processed
2016-05-24 09:59:28,222 DEBUG [org.keycloak.services] (default task-8) check execution: null requirement: ALTERNATIVE
2016-05-24 09:59:28,222 DEBUG [org.keycloak.services] (default task-8) Skip alternative execution
2016-05-24 09:59:28,223 DEBUG [org.keycloak.events] (default task-8) type=LOGIN, realmId=master, clientId=security-admin-console, userId=cf248811-6d82-47e7-bf9a-7b197fb988d0, ipAddress=192.168.33.1, auth_method=openid-connect, auth_type=code, response_type=code, redirect_uri=https://login.vagrant.v8/auth/admin/master/console/, consent=no_consent_required, code_id=7f9dbb5c-651a-4cc4-a248-91637c354fa5, response_mode=fragment, username=admin
2016-05-24 09:59:28,255 DEBUG [org.keycloak.services] (default task-8) Create login cookie - name: KEYCLOAK_IDENTITY, path: /auth/realms/master, max-age: -1
2016-05-24 09:59:28,256 DEBUG [org.keycloak.services] (default task-8) redirectAccessCode: state: 89b303fd-f77e-4b23-8de0-2c517304e671
2016-05-24 09:59:28,612 DEBUG [org.keycloak.services] (default task-11) AUTHENTICATE CLIENT
2016-05-24 09:59:28,612 DEBUG [org.keycloak.services] (default task-11) client authenticator: client-secret
2016-05-24 09:59:28,613 DEBUG [org.keycloak.services] (default task-11) client authenticator: client-jwt
2016-05-24 09:59:28,614 DEBUG [org.keycloak.services] (default task-11) KC-SERVICES0014: Failed client authentication: org.keycloak.authentication.AuthenticationFlowException: Client was not identified by any client authenticator
at org.keycloak.authentication.ClientAuthenticationFlow.processFlow(ClientAuthenticationFlow.java:101)
at org.keycloak.authentication.AuthenticationProcessor.authenticateClient(AuthenticationProcessor.java:673)
at org.keycloak.protocol.oidc.utils.AuthorizeClientUtil.authorizeClient(AuthorizeClientUtil.java:42)
at org.keycloak.protocol.oidc.endpoints.TokenEndpoint.checkClient(TokenEndpoint.java:163)
at org.keycloak.protocol.oidc.endpoints.TokenEndpoint.build(TokenEndpoint.java:117)
at sun.reflect.GeneratedMethodAccessor93.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:139)
at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTarget(ResourceMethodInvoker.java:295)
at org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:249)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:138)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:107)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:133)
at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:101)
at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:395)
at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:202)
at org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.service(ServletContainerDispatcher.java:221)
at org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:56)
at org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:51)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at io.undertow.servlet.handlers.ServletHandler.handleRequest(ServletHandler.java:85)
at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:129)
at org.keycloak.services.filters.KeycloakSessionServletFilter.doFilter(KeycloakSessionServletFilter.java:88)
at io.undertow.servlet.core.ManagedFilter.doFilter(ManagedFilter.java:60)
at io.undertow.servlet.handlers.FilterHandler$FilterChainImpl.doFilter(FilterHandler.java:131)
at io.undertow.servlet.handlers.FilterHandler.handleRequest(FilterHandler.java:84)
at io.undertow.servlet.handlers.security.ServletSecurityRoleHandler.handleRequest(ServletSecurityRoleHandler.java:62)
at io.undertow.servlet.handlers.ServletDispatchingHandler.handleRequest(ServletDispatchingHandler.java:36)
at org.wildfly.extension.undertow.security.SecurityContextAssociationHandler.handleRequest(SecurityContextAssociationHandler.java:78)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.servlet.handlers.security.SSLInformationAssociationHandler.handleRequest(SSLInformationAssociationHandler.java:131)
at io.undertow.servlet.handlers.security.ServletAuthenticationCallHandler.handleRequest(ServletAuthenticationCallHandler.java:57)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.security.handlers.AbstractConfidentialityHandler.handleRequest(AbstractConfidentialityHandler.java:46)
at io.undertow.servlet.handlers.security.ServletConfidentialityConstraintHandler.handleRequest(ServletConfidentialityConstraintHandler.java:64)
at io.undertow.security.handlers.AuthenticationMechanismsHandler.handleRequest(AuthenticationMechanismsHandler.java:60)
at io.undertow.servlet.handlers.security.CachedAuthenticatedSessionHandler.handleRequest(CachedAuthenticatedSessionHandler.java:77)
at io.undertow.security.handlers.NotificationReceiverHandler.handleRequest(NotificationReceiverHandler.java:50)
at io.undertow.security.handlers.AbstractSecurityContextAssociationHandler.handleRequest(AbstractSecurityContextAssociationHandler.java:43)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at org.wildfly.extension.undertow.security.jacc.JACCContextIdHandler.handleRequest(JACCContextIdHandler.java:61)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:43)
at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:284)
at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:263)
at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:81)
at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:174)
at io.undertow.server.Connectors.executeRootHandler(Connectors.java:202)
at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:793)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

2016-05-24 09:59:28,614 WARN  [org.keycloak.events] (default task-11) type=CODE_TO_TOKEN_ERROR, realmId=master, clientId=admin, userId=null, ipAddress=192.168.33.1, error=invalid_client_credentials, grant_type=authorization_code
2016-05-24 09:59:28,692 INFO  [io.undertow.request.dump] (default task-11) 
----------------------------REQUEST---------------------------
               URI=/auth/realms/master/protocol/openid-connect/token
 characterEncoding=null
     contentLength=229
       contentType=[application/x-www-form-urlencoded]
            cookie=KC_RESTART=eyJhbGciOiJIUzI1NiJ9.eyJjcyI6IjdmOWRiYjVjLTY1MWEtNGNjNC1hMjQ4LTkxNjM3YzM1NGZhNSIsImNpZCI6InNlY3VyaXR5LWFkbWluLWNvbnNvbGUiLCJwdHkiOiJvcGVuaWQtY29ubmVjdCIsInJ1cmkiOiJodHRwczovL2xvZ2luLnZhZ3JhbnQudjgvYXV0aC9hZG1pbi9tYXN0ZXIvY29uc29sZS8iLCJhY3QiOiJBVVRIRU5USUNBVEUiLCJub3RlcyI6eyJhY3Rpb25fa2V5IjoiYjg3OTM2Y2ItZTg1OC00NDhlLWJmMTAtNGNkYjQzMDFkMTg2IiwiYXV0aF90eXBlIjoiY29kZSIsImlzcyI6Imh0dHBzOi8vbG9naW4udmFncmFudC52OC9hdXRoL3JlYWxtcy9tYXN0ZXIiLCJyZXNwb25zZV90eXBlIjoiY29kZSIsInJlZGlyZWN0X3VyaSI6Imh0dHBzOi8vbG9naW4udmFncmFudC52OC9hdXRoL2FkbWluL21hc3Rlci9jb25zb2xlLyIsInN0YXRlIjoiODliMzAzZmQtZjc3ZS00YjIzLThkZTAtMmM1MTczMDRlNjcxIiwibm9uY2UiOiI1MzY0YTRhZS0wNzI3LTQwYWItOWEyMi0yN2ExZDQ5NjVmNTkiLCJyZXNwb25zZV9tb2RlIjoiZnJhZ21lbnQifX0.7Xnk589FgD_9RulKC4V48gn-056yVxa_DdIGm4Hpgj8
            cookie=KEYCLOAK_IDENTITY=eyJhbGciOiJSUzI1NiJ9.eyJqdGkiOiI5ZjRmNWU3OS02NWYxLTQ0M2ItOGI5YS1hZWI4NzZmYmNkNjIiLCJleHAiOjE0NjQxMTk5NjgsIm5iZiI6MCwiaWF0IjoxNDY0MDgzOTY4LCJpc3MiOiJodHRwczovL2xvZ2luLnZhZ3JhbnQudjgvYXV0aC9yZWFsbXMvbWFzdGVyIiwic3ViIjoiY2YyNDg4MTEtNmQ4Mi00N2U3LWJmOWEtN2IxOTdmYjk4OGQwIiwic2Vzc2lvbl9zdGF0ZSI6ImQzYjAxY2Q1LWQxNWEtNGQ2OS05MTQ3LWE1YTEzYWI4MjQxNiIsInJlc291cmNlX2FjY2VzcyI6e319.eiDWSR8eeIdX7AKZiWI3e9moUD9ZuD-h88XX31JRMio_FmqSvMgRHcZtkVjjrLnjcmWn7X4jIcITARk-TiywjfAKotkPYodgCyxsmln5LRQeV3SYXltCGWKmsFmq0BsMjQbfstjdR9V9Xh7daiyRNxLGcoYitEEOeKboMjbGwZw11jggaTRifpNYp-M9GtTBvHSJg6RFdmm0QTLeJm9OgOvBzdRlKwGTUcJeqJxuNWgd4XqPS5vbpliDsfuvekC8VFeDB_UGci1hCIEoMy0-tlpwPLP_xGbCd5L8XFKrr0shoGbQlyJjFZB49VtJlHLkd8F15mZQxJ1G9BWj_wKINw
            cookie=KEYCLOAK_SESSION=master/cf248811-6d82-47e7-bf9a-7b197fb988d0/d3b01cd5-d15a-4d69-9147-a5a13ab82416
            header=X-Real-Ip=192.168.33.1
            header=Accept-Encoding=gzip, deflate
            header=DNT=1
            header=Origin=https://login.vagrant.v8
            header=X-Forwarded-Port=443
            header=X-Forwarded-For=192.168.33.1
            header=Cookie=KC_RESTART=eyJhbGciOiJIUzI1NiJ9.eyJjcyI6IjdmOWRiYjVjLTY1MWEtNGNjNC1hMjQ4LTkxNjM3YzM1NGZhNSIsImNpZCI6InNlY3VyaXR5LWFkbWluLWNvbnNvbGUiLCJwdHkiOiJvcGVuaWQtY29ubmVjdCIsInJ1cmkiOiJodHRwczovL2xvZ2luLnZhZ3JhbnQudjgvYXV0aC9hZG1pbi9tYXN0ZXIvY29uc29sZS8iLCJhY3QiOiJBVVRIRU5USUNBVEUiLCJub3RlcyI6eyJhY3Rpb25fa2V5IjoiYjg3OTM2Y2ItZTg1OC00NDhlLWJmMTAtNGNkYjQzMDFkMTg2IiwiYXV0aF90eXBlIjoiY29kZSIsImlzcyI6Imh0dHBzOi8vbG9naW4udmFncmFudC52OC9hdXRoL3JlYWxtcy9tYXN0ZXIiLCJyZXNwb25zZV90eXBlIjoiY29kZSIsInJlZGlyZWN0X3VyaSI6Imh0dHBzOi8vbG9naW4udmFncmFudC52OC9hdXRoL2FkbWluL21hc3Rlci9jb25zb2xlLyIsInN0YXRlIjoiODliMzAzZmQtZjc3ZS00YjIzLThkZTAtMmM1MTczMDRlNjcxIiwibm9uY2UiOiI1MzY0YTRhZS0wNzI3LTQwYWItOWEyMi0yN2ExZDQ5NjVmNTkiLCJyZXNwb25zZV9tb2RlIjoiZnJhZ21lbnQifX0.7Xnk589FgD_9RulKC4V48gn-056yVxa_DdIGm4Hpgj8; KEYCLOAK_IDENTITY=eyJhbGciOiJSUzI1NiJ9.eyJqdGkiOiI5ZjRmNWU3OS02NWYxLTQ0M2ItOGI5YS1hZWI4NzZmYmNkNjIiLCJleHAiOjE0NjQxMTk5NjgsIm5iZiI6MCwiaWF0IjoxNDY0MDgzOTY4LCJpc3MiOiJodHRwczovL2xvZ2luLnZhZ3JhbnQudjgvYXV0aC9yZWFsbXMvbWFzdGVyIiwic3ViIjoiY2YyNDg4MTEtNmQ4Mi00N2U3LWJmOWEtN2IxOTdmYjk4OGQwIiwic2Vzc2lvbl9zdGF0ZSI6ImQzYjAxY2Q1LWQxNWEtNGQ2OS05MTQ3LWE1YTEzYWI4MjQxNiIsInJlc291cmNlX2FjY2VzcyI6e319.eiDWSR8eeIdX7AKZiWI3e9moUD9ZuD-h88XX31JRMio_FmqSvMgRHcZtkVjjrLnjcmWn7X4jIcITARk-TiywjfAKotkPYodgCyxsmln5LRQeV3SYXltCGWKmsFmq0BsMjQbfstjdR9V9Xh7daiyRNxLGcoYitEEOeKboMjbGwZw11jggaTRifpNYp-M9GtTBvHSJg6RFdmm0QTLeJm9OgOvBzdRlKwGTUcJeqJxuNWgd4XqPS5vbpliDsfuvekC8VFeDB_UGci1hCIEoMy0-tlpwPLP_xGbCd5L8XFKrr0shoGbQlyJjFZB49VtJlHLkd8F15mZQxJ1G9BWj_wKINw; KEYCLOAK_SESSION=master/cf248811-6d82-47e7-bf9a-7b197fb988d0/d3b01cd5-d15a-4d69-9147-a5a13ab82416
            header=Referer=https://login.vagrant.v8/auth/admin/master/console/
            header=Host=login.vagrant.v8
            header=Accept=*/*
            header=Accept-Language=en-US,en;q=0.8,de;q=0.6
            header=X-Original-To=192.168.33.80
            header=User-Agent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36
            header=Authorization=Basic YWRtaW46Y2hhbmdlbWU=
            header=X-Forwarded-Proto=https
            header=Content-Length=229
            header=Content-Type=application/x-www-form-urlencoded
            locale=[en_US, en, de]
            method=POST
          protocol=HTTP/1.1
       queryString=
        remoteAddr=/192.168.33.80:55892
        remoteHost=proxy.vagrant.v8
            scheme=https
              host=login.vagrant.v8
        serverPort=443
--------------------------RESPONSE--------------------------
     contentLength=123
       contentType=application/json
            header=X-Powered-By=Undertow/1
            header=Server=WildFly/10
            header=Content-Type=application/json
            header=Content-Length=123
            header=Date=Tue, 24 May 2016 09:59:28 GMT
            status=400


On Tue, May 24, 2016 at 9:49 PM, Stian Thorgersen <sthorger@redhat.com> wrote:
Did you add ProxyPeerAddressHandler filter? That's required for AJP connector, see http://keycloak.github.io/docs/userguide/keycloak-server/html/server-installation.html#proxy-address-forwarding

On 24 May 2016 at 11:48, Niels Bertram <nielsbne@gmail.com> wrote:
I am scratching my head with a specific setup problem which does not generate any usable error messages.

I am running a haproxy as load balancer in a vm in front an apache web server configured as reverse proxy connecting to the keycloak server via ajp in another VM.

client browser (192.168.33.1) 
      
       login.vagrant.v8 (192.168.33.80) aka proxy.vagrant.v8 is haproxy adds X-Forwarded-For X-Forwarded-Port X-Forwarded-Proto and X-Real-Ip

                kc01.vagrant.v8 (192.168.33.81) apache reverse proxies to wildfly on ajp port


Followed all the setup instructions in the documentation and if I connect to apache proxying through to keycloak everything works fine. All web resources are donwloaded fine however when I request a token exchange on /auth/realms/master/protocol/openid-connect/token I get a 400 response. The kc server log shows the corect IP address of the originating client and the request dump from wildfly also shows the correct  X-Forwarded-For header coming in. However the query string remoteAddr=/192.168.33.80:54672 which I believe is the one sent to the ajp connector shows some half valid IP address which is that of the load balancer. Did anyone come across this before? Looks like a bug of some sort.

The symptom is a endless loop trying to log into the admin panel.

Cheers
Niels


# cat standalone/log/server.log | grep -A 58 '2016-05-24 09:19:27,672'
2016-05-24 09:19:27,672 WARN  [org.keycloak.events] (default task-19) type=CODE_TO_TOKEN_ERROR, realmId=master, clientId=admin, userId=null, ipAddress=192.168.33.1, error=invalid_client_credentials, grant_type=authorization_code
2016-05-24 09:19:27,673 INFO  [io.undertow.request.dump] (default task-19)
----------------------------REQUEST---------------------------
               URI=/auth/realms/master/protocol/openid-connect/token
 characterEncoding=null
     contentLength=229
       contentType=[application/x-www-form-urlencoded]
            cookie=KC_RESTART=eyJhbGciOiJIUzI1NiJ9.eyJjcyI6IjM5YTVkNTlmLWMyNDYtNDkwZi04ZGZkLTZhYzVhNzgyZDI5ZCIsImNpZCI6InNlY3VyaXR5LWFkbWluLWNvbnNvbGUiLCJwdHkiOiJvcGVuaWQtY29ubmVjdCIsInJ1cmkiOiJodHRwczovL2xvZ2luLnZhZ3JhbnQudjgvYXV0aC9hZG1pbi9tYXN0ZXIvY29uc29sZS8iLCJhY3QiOiJBVVRIRU5USUNBVEUiLCJub3RlcyI6eyJhY3Rpb25fa2V5IjoiOGQ5ZGI4MzctMjY2Ni00NDcxLTk0ZDgtZmFkMmIxMjA3NDQxIiwiYXV0aF90eXBlIjoiY29kZSIsImlzcyI6Imh0dHBzOi8vbG9naW4udmFncmFudC52OC9hdXRoL3JlYWxtcy9tYXN0ZXIiLCJyZXNwb25zZV90eXBlIjoiY29kZSIsInJlZGlyZWN0X3VyaSI6Imh0dHBzOi8vbG9naW4udmFncmFudC52OC9hdXRoL2FkbWluL21hc3Rlci9jb25zb2xlLyIsInN0YXRlIjoiNTQ4ZDE5ZTUtMjlkMS00NTU2LWFjNjAtYjZjZTM1ZmJiMGU2Iiwibm9uY2UiOiI5OGY3NDFiYS03MmQwLTQ0ZDUtOGQ0ZC1jZTAxNzZhYjMyMmUiLCJyZXNwb25zZV9tb2RlIjoiZnJhZ21lbnQifX0.I0jI4nDhbYtKNrVjdlwjjBe5mtd0a8u6Dm7rQXwLE60
            cookie=KEYCLOAK_IDENTITY=eyJhbGciOiJSUzI1NiJ9.eyJqdGkiOiJhNjY5OWJkOS00MWQ4LTQyNWYtYjE5Ni04Y2QzNmJiZjBmNjQiLCJleHAiOjE0NjQxMTc1NjcsIm5iZiI6MCwiaWF0IjoxNDY0MDgxNTY3LCJpc3MiOiJodHRwczovL2xvZ2luLnZhZ3JhbnQudjgvYXV0aC9yZWFsbXMvbWFzdGVyIiwic3ViIjoiY2YyNDg4MTEtNmQ4Mi00N2U3LWJmOWEtN2IxOTdmYjk4OGQwIiwic2Vzc2lvbl9zdGF0ZSI6IjFiYTljODRlLTBlMzctNGE4Mi1hNDg0LWMyNWQyYzRhODBmYyIsInJlc291cmNlX2FjY2VzcyI6e319.E0vEe9XQJ_6IbDC_TEUfumQCJ0fS1_AOYsHh7svyGp16VC89sH9J1FQuLJfHYFVJlDTcE6o2ktLg0fLw2nLIdLOv-WXMseYr0KzudZveiLy1CZbRoPS9w9vlN-_EuXojiz0ORcyh90keUhqW5tMShccHvEaq_wpXOJQ6ITIglsgUXNhlSuEfpEcBy4CCqKQW98bRQiTKQOtoOfgc-Ez1RHR-7esTw-U22P_H-EMk23jI3nwuYGtqOn4Vvqb4-cHOzdyE_xaVWZxeteNKhU-RexfrMaHx1PSy3T796aY7gIljcqkxra-SA1dbOsRBawwlhJwFtojzBHEs1841gJ4bgg
            cookie=KEYCLOAK_SESSION=master/cf248811-6d82-47e7-bf9a-7b197fb988d0/1ba9c84e-0e37-4a82-a484-c25d2c4a80fc
            header=Accept=*/*
            header=Accept-Language=en-US,en;q=0.8,de;q=0.6
            header=Accept-Encoding=gzip, deflate
            header=DNT=1
            header=Origin=https://login.vagrant.v8
            header=X-Original-To=192.168.33.80
            header=User-Agent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36
            header=X-Forwarded-Proto=https
            header=X-Forwarded-Port=443
            header=X-Forwarded-For=192.168.33.1
            header=Content-Length=229
            header=Content-Type=application/x-www-form-urlencoded
            header=Cookie=KC_RESTART=eyJhbGciOiJIUzI1NiJ9.eyJjcyI6IjM5YTVkNTlmLWMyNDYtNDkwZi04ZGZkLTZhYzVhNzgyZDI5ZCIsImNpZCI6InNlY3VyaXR5LWFkbWluLWNvbnNvbGUiLCJwdHkiOiJvcGVuaWQtY29ubmVjdCIsInJ1cmkiOiJodHRwczovL2xvZ2luLnZhZ3JhbnQudjgvYXV0aC9hZG1pbi9tYXN0ZXIvY29uc29sZS8iLCJhY3QiOiJBVVRIRU5USUNBVEUiLCJub3RlcyI6eyJhY3Rpb25fa2V5IjoiOGQ5ZGI4MzctMjY2Ni00NDcxLTk0ZDgtZmFkMmIxMjA3NDQxIiwiYXV0aF90eXBlIjoiY29kZSIsImlzcyI6Imh0dHBzOi8vbG9naW4udmFncmFudC52OC9hdXRoL3JlYWxtcy9tYXN0ZXIiLCJyZXNwb25zZV90eXBlIjoiY29kZSIsInJlZGlyZWN0X3VyaSI6Imh0dHBzOi8vbG9naW4udmFncmFudC52OC9hdXRoL2FkbWluL21hc3Rlci9jb25zb2xlLyIsInN0YXRlIjoiNTQ4ZDE5ZTUtMjlkMS00NTU2LWFjNjAtYjZjZTM1ZmJiMGU2Iiwibm9uY2UiOiI5OGY3NDFiYS03MmQwLTQ0ZDUtOGQ0ZC1jZTAxNzZhYjMyMmUiLCJyZXNwb25zZV9tb2RlIjoiZnJhZ21lbnQifX0.I0jI4nDhbYtKNrVjdlwjjBe5mtd0a8u6Dm7rQXwLE60; KEYCLOAK_IDENTITY=eyJhbGciOiJSUzI1NiJ9.eyJqdGkiOiJhNjY5OWJkOS00MWQ4LTQyNWYtYjE5Ni04Y2QzNmJiZjBmNjQiLCJleHAiOjE0NjQxMTc1NjcsIm5iZiI6MCwiaWF0IjoxNDY0MDgxNTY3LCJpc3MiOiJodHRwczovL2xvZ2luLnZhZ3JhbnQudjgvYXV0aC9yZWFsbXMvbWFzdGVyIiwic3ViIjoiY2YyNDg4MTEtNmQ4Mi00N2U3LWJmOWEtN2IxOTdmYjk4OGQwIiwic2Vzc2lvbl9zdGF0ZSI6IjFiYTljODRlLTBlMzctNGE4Mi1hNDg0LWMyNWQyYzRhODBmYyIsInJlc291cmNlX2FjY2VzcyI6e319.E0vEe9XQJ_6IbDC_TEUfumQCJ0fS1_AOYsHh7svyGp16VC89sH9J1FQuLJfHYFVJlDTcE6o2ktLg0fLw2nLIdLOv-WXMseYr0KzudZveiLy1CZbRoPS9w9vlN-_EuXojiz0ORcyh90keUhqW5tMShccHvEaq_wpXOJQ6ITIglsgUXNhlSuEfpEcBy4CCqKQW98bRQiTKQOtoOfgc-Ez1RHR-7esTw-U22P_H-EMk23jI3nwuYGtqOn4Vvqb4-cHOzdyE_xaVWZxeteNKhU-RexfrMaHx1PSy3T796aY7gIljcqkxra-SA1dbOsRBawwlhJwFtojzBHEs1841gJ4bgg; KEYCLOAK_SESSION=master/cf248811-6d82-47e7-bf9a-7b197fb988d0/1ba9c84e-0e37-4a82-a484-c25d2c4a80fc
            header=Host=login.vagrant.v8
            locale=[en_US, en, de]
            method=POST
          protocol=HTTP/1.1
       queryString=
        remoteAddr=/192.168.33.80:54672
        remoteHost=proxy.vagrant.v8
            scheme=https
              host=login.vagrant.v8
        serverPort=443
--------------------------RESPONSE--------------------------
     contentLength=123
       contentType=application/json
            header=X-Powered-By=Undertow/1
            header=Server=WildFly/10
            header=Content-Type=application/json
            header=Content-Length=123
            header=Date=Tue, 24 May 2016 09:19:27 GMT
            status=400


_______________________________________________
keycloak-user mailing list
keycloak-user@lists.jboss.org
https://lists.jboss.org/mailman/listinfo/keycloak-user