Problem running keycloak cluster on EC2 with S3_ping

Friday, 18 December 2015

hi,

I'm having trouble configuring a Keycloak cluster running on AWS' EC2.
The database configuration is OK no problem, but i can't manage to get the
invalidation cache working correctly.
I configured Infinispan to work with S3_ping plugin (the relevant part of
my configuration is below).

When i run both server, the connection with the database is Ok, but the
infinispan logs look like this :
On Server 1 :
...
11:00:17,592 INFO  [stdout] (MSC service thread 1-1) GMS:
address=ip-10-1-7-103, cluster=ee, physical address=10.1.7.103:7600
...
11:00:18,057 INFO
 [org.infinispan.remoting.transport.jgroups.JGroupsTransport]
(ServerService Thread Pool -- 62) ISPN000094: Received new cluster view for
channel keycloak: [ip-10-1-7-103|0] (1) [ip-10-1-7-103]
...

On Server 2 :
...
11:03:41,159 INFO  [stdout] (MSC service thread 1-1) GMS:
address=ip-10-1-1-245, cluster=ee, physical address=10.1.1.245:7600
...
11:03:41,783 INFO
 [org.infinispan.remoting.transport.jgroups.JGroupsTransport]
(ServerService Thread Pool -- 62) ISPN000094: Received new cluster view for
channel keycloak: [ip-10-1-1-245|0] (1) [ip-10-1-1-245]
...

In my S3 bucket, i have 2 files created :
402ea329-c135-f1e9-2782-02768779e02f.ip-10-1-1-245.list
a584321f-408b-b2ae-e2dd-d19333db96c4.ip-10-1-7-103.list

And the content of the files is like this :
File 1 :
ip-10-1-1-245 402ea329-c135-f1e9-2782-02768779e02f 10.1.1.245:7600 T

File 2 :
ip-10-1-7-103 a584321f-408b-b2ae-e2dd-d19333db96c4 10.1.7.103:7600 T

When i read the logs, it looks like the infinispan's cache can't contact
each other.
I double check my network config, and i tried connecting from one server to
the other using nc (like this: nc -vvv 10.1.7.103 7600) and this works
fine.

Is there a way to check the infinispan status of the servers?
Do you guys got any clue on how to make this works?

Thank you,
Charles-Edouard

My config looks like this :

- Standalone-ha.xml
...
<datasources>
                <driver name="postgresql"
module="org.postgresql">

 <datasource-class>org.postgresql.Driver</datasource-class>

 <xa-datasource-class>org.postgresql.xa.PGXADataSource</xa-datasource-class>
                </driver>
                <datasource jndi-name="java:jboss/datasources/PgDskeycloak"
pool-name="PgDskeycloak" enabled="true"
use-java-context="true">

<connection-url>jdbc:postgresql://****:5432/keycloak?ApplicationName=keycloak</connection-url>
                    <driver>postgresql</driver>
                    <pool>
                        <min-pool-size>5</min-pool-size>
                        <initial-pool-size>5</initial-pool-size>
                        <max-pool-size>100</max-pool-size>
                        <prefill>true</prefill>
                    </pool>
                    <validation>
                        <valid-connection-checker

class-name="org.jboss.jca.adapters.jdbc.extensions.postgres.PostgreSQLValidConnectionChecker"></valid-connection-checker>
                        <exception-sorter

class-name="org.jboss.jca.adapters.jdbc.extensions.postgres.PostgreSQLExceptionSorter"></exception-sorter>
                    </validation>
                    <security>
                        <user-name>****</user-name>
                        <password>****</password>
                    </security>
                </datasource>
...
            <stacks default="tcp">
                <stack name="udp">
                    <transport type="UDP"
socket-binding="jgroups-udp"/>
                    <protocol type="PING"/>
                    <protocol type="MERGE3"/>
                    <protocol type="FD_SOCK"
socket-binding="jgroups-udp-fd"/>
                    <protocol type="FD_ALL"/>
                    <protocol type="VERIFY_SUSPECT"/>
                    <protocol type="pbcast.NAKACK2"/>
                    <protocol type="UNICAST3"/>
                    <protocol type="pbcast.STABLE"/>
                    <protocol type="pbcast.GMS"/>
                    <protocol type="UFC"/>
                    <protocol type="MFC"/>
                    <protocol type="FRAG2"/>
                    <protocol type="RSVP"/>
                </stack>
                <stack name="tcp">
                    <transport type="TCP"
socket-binding="jgroups-tcp"/>
                    <protocol type="S3_PING" >
                        <property name="location">****</property>
                        <property name="access_key">****</property>
                        <property
name="secret_access_key">****</property>
                    </protocol>
                    <!-- <protocol type="MPING"
socket-binding="jgroups-mping"/> -->
                    <protocol type="MERGE3"/>
                    <protocol type="FD_SOCK"
socket-binding="jgroups-tcp-fd"/>
                    <protocol type="FD"/>
                    <protocol type="VERIFY_SUSPECT"/>
                    <protocol type="pbcast.NAKACK2"/>
                    <protocol type="UNICAST3"/>
                    <protocol type="pbcast.STABLE"/>
                    <protocol type="pbcast.GMS"/>
                    <protocol type="MFC"/>
                    <protocol type="FRAG2"/>
                    <protocol type="RSVP"/>
                </stack>
...
    <interfaces>
        <interface name="management">
                <nic name="eth0"/>
        </interface>
        <interface name="public">
            <nic name="eth0"/>
        </interface>
        <!-- TODO - only show this if the jacorb subsystem is added  -->
        <interface name="unsecure">
            <!--
              ~  Used for IIOP sockets in the standard configuration.
              ~                  To secure JacORB you need to setup SSL
              -->
            <nic name="eth0"/>
        </interface>
    </interfaces>

- keycloak-server.json
{
    "providers": [
        "classpath:${jboss.server.config.dir}/providers/*"
    ],

    "admin": {
        "realm": "master"
    },

    "eventsStore": {
        "provider": "jpa",
        "jpa": {
            "exclude-events": [ "REFRESH_TOKEN" ]
        }
    },

    "realm": {
        "provider": "jpa"
    },

    "user": {
        "provider": "jpa"
    },

    "userSessionPersister": {
        "provider": "jpa"
    },

    "timer": {
        "provider": "basic"
    },

    "theme": {
        "default": "keycloak",
        "staticMaxAge": 2592000,
        "cacheTemplates": true,
        "cacheThemes": true,
        "folder": {
          "dir": "${jboss.server.config.dir}/themes"
        }
    },

    "scheduled": {
        "interval": 900
    },

    "connectionsHttpClient": {
        "default": {
            "disable-trust-manager": true
        }
    },

    "connectionsJpa": {
        "default": {
            "dataSource": "java:jboss/datasources/PgDskeycloak",
            "databaseSchema": "update"
        }
    },

    "connectionsInfinispan": {
        "default" : {
            "cacheContainer" : "java:jboss/infinispan/Keycloak"
        }
    }
}

CHARLES-EDOUARD GAGNAIRE
SysAdmin
c.gagnaire(a)kreactive.com
p. 06.27.80.28.53LYON "Le Capitole"
97, cours Gambetta
69481 Lyon Cedex 03

PARIS
16, rue de Turbigo
75002 Paris
[image: Kreactive] <http://www.kreactive.com/>

[image: Facebook] <https://www.facebook.com/kreactive> [image: Twitter]
<https://twitter.com/kreactive>

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014