<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
    <head> 
        <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> 
        <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0"> 
        <base href="https://issues.jboss.org"> 
        <title>Message Title</title> 
    </head> 
    <body class="jira" style="color: #333333; font-family: Arial, sans-serif; font-size: 14px; line-height: 1.429"> 
        <table id="background-table" cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; background-color: #f5f5f5; border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt" bgcolor="#f5f5f5"> 
            <!-- header here --> 
            <tbody>
                <tr> 
                    <td id="header-pattern-container" style="padding: 0px; border-collapse: collapse; padding: 10px 20px"> 
                        <table id="header-pattern" cellspacing="0" cellpadding="0" border="0" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt"> 
                            <tbody>
                                <tr> 
                                    <td id="header-avatar-image-container" valign="top" style="padding: 0px; border-collapse: collapse; vertical-align: top; width: 32px; padding-right: 8px" width="32"> <img id="header-avatar-image" class="image_fix" src="https://static.jboss.org/developer/gravatar/0d5a0dc52688f185ff93275f1ec9e7ff?d=mm&amp;s=48" height="32" width="32" border="0" style="border-radius: 3px; vertical-align: top"> </td> 
                                    <td id="header-text-container" valign="middle" style="padding: 0px; border-collapse: collapse; vertical-align: middle; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 1px"> <a class="user-hover" rel="peter.braun" id="email_peter.braun" href="https://issues.jboss.org/secure/ViewProfile.jspa?name=peter.braun" style="color:#3b73af;; color: #3b73af; text-decoration: none">Peter Braun</a> <strong>commented</strong> on <a href="https://issues.jboss.org/browse/AEROGEAR-1883" style="color: #3b73af; text-decoration: none"><img src="cid:jira-generated-image-avatar-eb135b9f-60b4-452f-a553-a95c340966e1" height="16" width="16" border="0" align="absmiddle" alt="Task"> AEROGEAR-1883</a> </td> 
                                </tr> 
                            </tbody>
                        </table> </td> 
                </tr> 
                <tr> 
                    <td id="email-content-container" style="padding: 0px; border-collapse: collapse; padding: 0 20px"> 
                        <table id="email-content-table" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; border-spacing: 0; border-collapse: separate"> 
                            <tbody>
                                <tr> 
                                    <!-- there needs to be content in the cell for it to render in some clients --> 
                                    <td class="email-content-rounded-top mobile-expand" style="padding: 0px; border-collapse: collapse; color: #ffffff; padding: 0 15px 0 16px; height: 15px; background-color: #ffffff; border-left: 1px solid #cccccc; border-top: 1px solid #cccccc; border-right: 1px solid #cccccc; border-bottom: 0; border-top-right-radius: 5px; border-top-left-radius: 5px; height: 10px; line-height: 10px; padding: 0 15px 0 16px; mso-line-height-rule: exactly" height="10" bgcolor="#ffffff">&nbsp;</td> 
                                </tr> 
                                <tr> 
                                    <td class="email-content-main mobile-expand " style="padding: 0px; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff" bgcolor="#ffffff"> 
                                        <table class="page-title-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt"> 
                                            <tbody>
                                                <tr> 
                                                    <td style="vertical-align: top;; padding: 0px; border-collapse: collapse; padding-right: 5px; font-size: 20px; line-height: 30px; mso-line-height-rule: exactly" class="page-title-pattern-header-container"> <span class="page-title-pattern-header" style="font-family: Arial, sans-serif; padding: 0; font-size: 20px; line-height: 30px; mso-text-raise: 2px; mso-line-height-rule: exactly; vertical-align: middle"> <a href="https://issues.jboss.org/browse/AEROGEAR-1883" style="color: #3b73af; text-decoration: none">Re: Investigate the state of Keycloak Metrics</a> </span> </td> 
                                                </tr> 
                                            </tbody>
                                        </table> </td> 
                                </tr> 
                                <tr> 
                                    <td id="text-paragraph-pattern-top" class="email-content-main mobile-expand  comment-top-pattern" style="padding: 0px; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff; border-bottom: none; padding-bottom: 0" bgcolor="#ffffff"> 
                                        <table class="text-paragraph-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 2px"> 
                                            <tbody>
                                                <tr> 
                                                    <td class="text-paragraph-pattern-container mobile-resize-text " style="padding: 0px; border-collapse: collapse; padding: 0 0 10px 0"> <p style="margin: 10px 0 0 0; margin-top: 0">There are a number of ways that allow us to extract metrics from Keycloak. Picking the metrics that we are interested in first will help us to make a decision here. The following are a list of Options that i found (and also the current implementation):</p> 
                                                        <ol> 
                                                            <li>Keycloak SPI + Sidecar Exporter (current implementation)</li> 
                                                            <li>Keycloak SPI that adds a custom REST endpoint</li> 
                                                            <li>Leveraging Java JMX (Java Management Extensions)</li> 
                                                            <li>Implement a Metrics Endpoint and get it accepted to Keycloak</li> 
                                                        </ol> <p style="margin: 10px 0 0 0"><b>Keycloak SPI + Sidecar Exporter (current implementation)</b></p> <p style="margin: 10px 0 0 0">Keycloak allows a good amount of extensibility through Service Provider Interfaces (SPI). Currently we are hooking into Keycloak's internal events by implementing a custom <em>EventListenerProvider</em>. This provider receives internal events such as <em>User Created</em>, <em>Login Succeeded</em>, etc. and writes them to the file system (one file per event type, the file contains the number of times the event happend).</p> <p style="margin: 10px 0 0 0">Another Service is running in the same POD that implements a small http server and exports the information from the filesystem as metrics data. An existing project is used for this: <a href="https://github.com/larscheid-schmitzhermes/keycloak-monitoring-prometheus/blob/master/src/main/java/com/larscheidschmitzhermes/keycloak/events/MonitoringEventListenerProvider.java" class="external-link" rel="nofollow" style="color: #3b73af; text-decoration: none">https://github.com/larscheid-schmitzhermes/keycloak-monitoring-prometheus/blob/master/src/main/java/com/larscheidschmitzhermes/keycloak/events/MonitoringEventListenerProvider.java</a></p> <p style="margin: 10px 0 0 0">Pros:<br> + Metrics based on internal Keycloak events<br> Cons:</p> 
                                                        <ul class="alternate" type="square"> 
                                                            <li>There seem to be better ways to do it</li> 
                                                            <li>No CPU or Memory Metrics</li> 
                                                            <li>Requires extra service</li> 
                                                        </ul> <p style="margin: 10px 0 0 0"><b>Keycloak SPI that adds a custom REST endpoint</b></p> <p style="margin: 10px 0 0 0">Keycloak SPIs are pretty powerful and can be used to add custom REST endpoints (see <a href="http://www.keycloak.org/docs/3.0/server_development/topics/extensions.html" class="external-link" rel="nofollow" style="color: #3b73af; text-decoration: none">http://www.keycloak.org/docs/3.0/server_development/topics/extensions.html</a>). This is done by implementing the <em>RealmResourceProvider</em> and returning a JAX-RS resource. There is even a way for the provider to make use of 3rd party libraries, which allows us to use the official Prometheus Java Client (<a href="https://github.com/prometheus/client_java" class="external-link" rel="nofollow" style="color: #3b73af; text-decoration: none">https://github.com/prometheus/client_java</a>) to collect metrics data based on Keycloak Events. There is also a way to extract memory usage (in Java Runtime.getRuntime() seems to offer some data) and there might be ways to get CPU usage as well.</p> <p style="margin: 10px 0 0 0">Pros:<br> + Metrics of internal Keycloak events<br> + We can use the official Prometheus Java Client lib<br> + No extra service (SPI deployed to JBoss)<br> Cons:</p> 
                                                        <ul class="alternate" type="square"> 
                                                            <li>Not ideal to get memory and cpu usage but it might be possible</li> 
                                                        </ul> <p style="margin: 10px 0 0 0"><b>Leveraging Java JMX (Java Management Extensions)</b></p> <p style="margin: 10px 0 0 0">Java Enterprise Edition offers a System called JMX that can be used for instrumentation and metrics. Since we are running Keycloak inside JBoss we have the option to make use of this. The way it works is quite different from the other approaches. In JMX you use a <em>Java Agent</em>, a jar that implements a <em>premain</em> method and gets loaded before any other application is run on the server. </p> <p style="margin: 10px 0 0 0">The Agent has access to so called <em>MBeans</em> that represent resources inside the Runtime and expose information such as fine grained Memory, CPU usage and other metrics.</p> <p style="margin: 10px 0 0 0">Luckily for us there is already a Prometheus exporter that can be started as a Java Agent and hooks into JMX. It starts a server on a configurable port that exposes the collected metrics (<a href="https://github.com/prometheus/jmx_exporter" class="external-link" rel="nofollow" style="color: #3b73af; text-decoration: none">https://github.com/prometheus/jmx_exporter</a>).</p> <p style="margin: 10px 0 0 0">Pros:<br> + Good way to get CPU and Memory metrics<br> + Prometheus offers a JMX Agent implementation that we can use<br> + No extra service (Agent running in JBoss)<br> Cons:</p> 
                                                        <ul class="alternate" type="square"> 
                                                            <li>Likely not good for internal Keycloak Metrics</li> 
                                                        </ul> <p style="margin: 10px 0 0 0"><b>Implement a Metrics Endpoint and get it accepted to Keycloak</b></p> <p style="margin: 10px 0 0 0">There is a Keycloak JIRA about this but no results so far (<a href="https://issues.jboss.org/browse/KEYCLOAK-1578" class="external-link" rel="nofollow" style="color: #3b73af; text-decoration: none">https://issues.jboss.org/browse/KEYCLOAK-1578</a>). The Idea is that we implement the metrics collection and endpoint, send a PR and hope to get it accepted into Keycloak. One problem is that Prometheus expects a certain format of metrics that is best implemented by using the official Prometheus Java client libs. So we would have to add a new dependency and discuss the metrics format with the Keycloak team. If it gets accepted we would have to wait for the next Keycloak release (or use our own build).</p> <p style="margin: 10px 0 0 0">Pros:<br> + No extra service<br> + No extra jars required, all built into Keycloak<br> Cons:</p> 
                                                        <ul class="alternate" type="square"> 
                                                            <li>Depends on a decision of the Keycloak Team to accept our PR and Prometheus metrics format</li> 
                                                        </ul> </td> 
                                                </tr> 
                                            </tbody>
                                        </table> </td> 
                                </tr> 
                                <tr> 
                                    <td class="email-content-main mobile-expand " style="padding: 0px; border-collapse: collapse; border-left: 1px solid #cccccc; border-right: 1px solid #cccccc; border-top: 0; border-bottom: 0; padding: 0 15px 0 16px; background-color: #ffffff" bgcolor="#ffffff"> 
                                        <table id="actions-pattern" cellspacing="0" cellpadding="0" border="0" width="100%" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 1px"> 
                                            <tbody>
                                                <tr> 
                                                    <td id="actions-pattern-container" valign="middle" style="padding: 0px; border-collapse: collapse; padding: 10px 0 10px 24px; vertical-align: middle; padding-left: 0"> 
                                                        <table align="left" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt"> 
                                                            <tbody>
                                                                <tr> 
                                                                    <td class="actions-pattern-action-icon-container" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 0; vertical-align: middle"> <a href="https://issues.jboss.org/browse/AEROGEAR-1883#add-comment" target="_blank" title="Add Comment" style="color: #3b73af; text-decoration: none"> <img class="actions-pattern-action-icon-image" src="cid:jira-generated-image-static-comment-icon-5c6a6917-51ab-480f-82dd-55d68cce5ef3" alt="Add Comment" title="Add Comment" height="16" width="16" border="0" style="vertical-align: middle"> </a> </td> 
                                                                    <td class="actions-pattern-action-text-container" style="padding: 0px; border-collapse: collapse; font-family: Arial, sans-serif; font-size: 14px; line-height: 20px; mso-line-height-rule: exactly; mso-text-raise: 4px; padding-left: 5px"> <a href="https://issues.jboss.org/browse/AEROGEAR-1883#add-comment" target="_blank" title="Add Comment" style="color: #3b73af; text-decoration: none">Add Comment</a> </td> 
                                                                </tr> 
                                                            </tbody>
                                                        </table> </td> 
                                                </tr> 
                                            </tbody>
                                        </table> </td> 
                                </tr> 
                                <!-- there needs to be content in the cell for it to render in some clients --> 
                                <tr> 
                                    <td class="email-content-rounded-bottom mobile-expand" style="padding: 0px; border-collapse: collapse; color: #ffffff; padding: 0 15px 0 16px; height: 5px; line-height: 5px; background-color: #ffffff; border-top: 0; border-left: 1px solid #cccccc; border-bottom: 1px solid #cccccc; border-right: 1px solid #cccccc; border-bottom-right-radius: 5px; border-bottom-left-radius: 5px; mso-line-height-rule: exactly" height="5" bgcolor="#ffffff">&nbsp;</td> 
                                </tr> 
                            </tbody>
                        </table> </td> 
                </tr> 
                <tr> 
                    <td id="footer-pattern" style="padding: 0px; border-collapse: collapse; padding: 12px 20px"> 
                        <table id="footer-pattern-container" cellspacing="0" cellpadding="0" border="0" style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt"> 
                            <tbody>
                                <tr> 
                                    <td id="footer-pattern-text" class="mobile-resize-text" width="100%" style="padding: 0px; border-collapse: collapse; color: #999999; font-size: 12px; line-height: 18px; font-family: Arial, sans-serif; mso-line-height-rule: exactly; mso-text-raise: 2px"> This message was sent by Atlassian JIRA <span id="footer-build-information">(v7.5.0#75005-<span title="fd8c849d4e278dd8bbaccc61e707a716ad697024" data-commit-id="fd8c849d4e278dd8bbaccc61e707a716ad697024}">sha1:fd8c849</span>)</span> </td> 
                                    <td id="footer-pattern-logo-desktop-container" valign="top" style="padding: 0px; border-collapse: collapse; padding-left: 20px; vertical-align: top"> 
                                        <table style="border-collapse: collapse; mso-table-lspace: 0pt; mso-table-rspace: 0pt"> 
                                            <tbody>
                                                <tr> 
                                                    <td id="footer-pattern-logo-desktop-padding" style="padding: 0px; border-collapse: collapse; padding-top: 3px"> <img id="footer-pattern-logo-desktop" src="cid:jira-generated-image-static-footer-desktop-logo-d1ca08f0-2cfc-4b10-ae4e-e3dda0cd8113" alt="Atlassian logo" title="Atlassian logo" width="169" height="36" class="image_fix"> </td> 
                                                </tr> 
                                            </tbody>
                                        </table> </td> 
                                </tr> 
                            </tbody>
                        </table> </td> 
                </tr> 
            </tbody>
        </table>   
    </body>
</html>