Add Prometheus support to IdP DS

wso2 · Mar 5, 2024 · 3b11097 · 3b11097
1 parent 63f20ca
commit 3b11097
Show file tree

Hide file tree

Showing 13 changed files with 129 additions and 15 deletions.
diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml
@@ -32,8 +32,8 @@ data:
     metricsReporter="prometheus"
     [ballerinax.prometheus]
     port=18008
-  {{if .Values.wso2.apk.metrics.configDSBalPort}}
-    host="{{ .Values.wso2.apk.metrics.configDSBalPort}}"
+  {{if .Values.wso2.apk.metrics.configDSBalHost}}
+    host="{{ .Values.wso2.apk.metrics.configDSBalHost}}"
   {{else}} 
     host="0.0.0.0"
   {{end}} 

diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml
@@ -25,7 +25,19 @@ data:
     [ballerina.log]
     level = "DEBUG"
     [ballerina.http]
-    traceLogConsole = true
+    traceLogConsole = false
+    {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}}
+    [ballerina.observe]
+    metricsEnabled=true
+    metricsReporter="prometheus"
+    [ballerinax.prometheus]
+    port=18011
+    {{if .Values.wso2.apk.metrics.idpDSBalHost}}
+    host="{{ .Values.wso2.apk.metrics.idpDSBalHost}}"
+    {{else}} 
+    host="0.0.0.0"
+    {{end}} 
+    {{end}} 
     # Sample configurations      
     [wso2.idp_domain_service.datasourceConfiguration]
     description = "Database for idp"

diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml
@@ -51,6 +51,12 @@ spec:
           ports:
             - containerPort: 9443
               protocol: "TCP"
+            {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }}
+            - containerPort: 18011
+              protocol: "TCP"
+            - containerPort: 18012
+              protocol: "TCP"
+            {{ end }}
           readinessProbe:
             httpGet:
               path: /health
@@ -76,6 +82,9 @@ spec:
             - mountPath: /home/wso2apk/idp/security/wso2carbon.pem
               name: idp-signing-keystore-volume
               subPath: wso2carbon.pem
+            - name: prometheus-jmx-config-volume	
+              mountPath: /tmp/metrics/prometheus-jmx-config-idpds.yml	
+              subPath: prometheus-jmx-config-idpds.yml
             - name: idp-ds-tls-volume
               mountPath: /home/wso2apk/idp/security/idp.key
             {{- if and .Values.idp.idpds.configs .Values.idp.idpds.configs.tls }}
@@ -111,6 +120,9 @@ spec:
         - name: config-toml-volume
           configMap:
             name: {{ template "apk-helm.resource.prefix" . }}-idp-ds-configmap
+        - name: prometheus-jmx-config-volume	
+          configMap: 	
+            name: prometheus-jmx-config-idpds
         - name: idp-signing-keystore-volume
           secret: 
             secretName: {{ template "apk-helm.resource.prefix" . }}-idp-signing-keystore-secret

diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml
@@ -0,0 +1,31 @@
+# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved.
+#
+# WSO2 LLC. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-jmx-config-idpds
+data:
+  prometheus-jmx-config-idpds.yml: |
+    lowercaseOutputName: true
+    lowercaseOutputLabelNames: true
+    rules:
+      # OS related metrics
+      - pattern: 'java.lang<type=OperatingSystem><>(\w+): (.*)'
+        name: os_$1
+        help: Operating System $1
+        attrNameSnakeCase: true
+        type: GAUGE
diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml
@@ -251,8 +251,10 @@ wso2:
 #                hostnameVerifier: "AllowAll"
 
     metrics:
-      enabled: true
-      # configDSBalPort: 0.0.0.0
+      enabled: false
+      # configDSBalHost: 0.0.0.0
+      # idpDSBalHost: 0.0.0.0
+
 idp:
   enabled: true
   listener:

diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template
@@ -592,6 +592,14 @@ wso2:
               keepAliveTime: 600
               # -- Queue size of the worker threads
               queueSize: 1000
+    metrics:
+      # -- Enable Prometheus metrics
+      enabled: false
+      # -- Configure the host for exposing the config ds ballerina metrics
+      # configDSBalHost: 0.0.0.0
+      # -- Configure the host for exposing the idp ds ballerina metrics.
+      # idpDSBalHost: 0.0.0.0
+
 idp:
   # -- Enable Non production identity server
   enabled: true

diff --git a/idp/idp-domain-service/ballerina/Ballerina.toml b/idp/idp-domain-service/ballerina/Ballerina.toml
@@ -9,5 +9,10 @@ groupId = "org.postgresql"
 artifactId = "postgresql"
 version = "42.2.20"
 
+[[platform.java11.dependency]]
+groupId = "io.prometheus.jmx"
+artifactId = "jmx_prometheus_javaagent"
+version = "0.20.0"
+
 [build-options]
 observabilityIncluded = true
diff --git a/idp/idp-domain-service/ballerina/Dependencies.toml b/idp/idp-domain-service/ballerina/Dependencies.toml
@@ -375,6 +375,21 @@ modules = [
 	{org = "ballerinax", packageName = "postgresql", moduleName = "postgresql"}
 ]
 
+[[package]]
+org = "ballerinax"
+name = "prometheus"
+version = "0.2.0"
+dependencies = [
+	{org = "ballerina", name = "http"},
+	{org = "ballerina", name = "io"},
+	{org = "ballerina", name = "lang.string"},
+	{org = "ballerina", name = "observe"},
+	{org = "ballerina", name = "regex"}
+]
+modules = [
+	{org = "ballerinax", packageName = "prometheus", moduleName = "prometheus"}
+]
+
 [[package]]
 org = "wso2"
 name = "idp_domain_service"
@@ -392,7 +407,8 @@ dependencies = [
 	{org = "ballerina", name = "url"},
 	{org = "ballerina", name = "uuid"},
 	{org = "ballerinai", name = "observe"},
-	{org = "ballerinax", name = "postgresql"}
+	{org = "ballerinax", name = "postgresql"},
+	{org = "ballerinax", name = "prometheus"}
 ]
 modules = [
 	{org = "wso2", packageName = "idp_domain_service", moduleName = "idp_domain_service"}

diff --git a/idp/idp-domain-service/ballerina/Init.bal b/idp/idp-domain-service/ballerina/Init.bal
@@ -20,6 +20,7 @@ import ballerina/log;
 import ballerina/sql;
 import ballerina/http;
 import ballerina/jwt;
+import ballerinax/prometheus as _;
 
 configurable IDPConfiguration & readonly idpConfiguration = ?;
 final postgresql:Client|sql:Error dbClient;

diff --git a/idp/idp-domain-service/docker/Dockerfile b/idp/idp-domain-service/docker/Dockerfile
@@ -72,6 +72,7 @@ RUN groupadd --system -g ${USER_GROUP_ID} ${USER_GROUP} \
 
 ADD idp ${USER_HOME}/idp
 COPY docker-entrypoint.sh ${USER_HOME}
+COPY /lib/jmx_prometheus_javaagent-0.20.0.jar /home/${USER}/lib/
 
 RUN chown -R ${USER} ${USER_HOME}/idp \
     && chown ${USER} /home/${USER}/docker-entrypoint.sh \

diff --git a/idp/idp-domain-service/docker/build.gradle b/idp/idp-domain-service/docker/build.gradle
@@ -18,20 +18,45 @@
 
 apply from: "$projectDir/../../../common-gradle-scripts/docker.gradle"
 apply from: "$rootDir/../../common-gradle-scripts/copy.gradle"
+apply from: "$projectDir/../../../common-gradle-scripts/java.gradle"
 
+dependencies {
+    implementation libs.prometheus
+}
+
+tasks.register('copy_dependencies', Copy) {
+    dependsOn create_lib
+    from configurations.default
+    into "$projectDir/../docker/lib"
+    finalizedBy build
+}
+
+tasks.register('create_lib') {
+    dependsOn delete_lib
+    finalizedBy copy_dependencies
+}
+
+tasks.register('delete_lib', Delete) {
+    delete "$projectDir/lib"
+    finalizedBy create_lib
+}
+
+tasks.named('build').configure {
+    dependsOn copy_dependencies
+}
 
 tasks.named('copy_dist').configure{
     finalizedBy docker_build
 }
 
-task build{
+task buildDockerImage{
     group 'build'
     description 'Build docker image'
     dependsOn 'copy_dist'
     dependsOn 'docker_build'
 }
 
-build.configure{
+buildDockerImage.configure{
     mustRunAfter(":ballerina:build")
     dependsOn(":ballerina:build")
     }
diff --git a/idp/idp-domain-service/docker/idp/idp.sh b/idp/idp-domain-service/docker/idp/idp.sh
@@ -98,9 +98,10 @@ fi
 echo "Using Java memory options: $JVM_MEM_OPTS"
 
 $JAVACMD \
-    $JVM_MEM_OPTS \
-    $JAVA_OPTS \
-    -classpath "$CLASSPATH" \
-    -Djava.io.tmpdir="$IDP_HOME/tmp" \
-    -jar idp_domain_service.jar $*
-    status=$?
+  $JVM_MEM_OPTS \
+  $JAVA_OPTS \
+  -classpath "$CLASSPATH" \
+  -Djava.io.tmpdir="$IDP_HOME/tmp" \
+  -javaagent:/home/wso2apk/jmx_prometheus_javaagent-0.20.0.jar=18012:/tmp/metrics/prometheus-jmx-config-idpds.yml \
+  -jar idp_domain_service.jar $*
+status=$?
diff --git a/runtime/config-deployer-service/docker/config-deployer/config.sh b/runtime/config-deployer-service/docker/config-deployer/config.sh
@@ -103,6 +103,6 @@ $JAVACMD \
   $JAVA_OPTS \
   -classpath "$CLASSPATH" \
   -Djava.io.tmpdir="$RUNTIME_HOME/tmp" \
-  -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18010:/tmp/metrics/prometheus-jmx-config-configds.yml \
+  -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18009:/tmp/metrics/prometheus-jmx-config-configds.yml \
   -jar config_deployer_service.jar $*
 status=$?