Add Prometheus support to Config DS

wso2 · Feb 29, 2024 · 218b769 · 218b769
1 parent 2dfd1e4
commit 218b769
Show file tree

Hide file tree

Showing 11 changed files with 153 additions and 18 deletions.
diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml
@@ -25,17 +25,17 @@ data:
     [ballerina.log]
     level = "DEBUG"
     [ballerina.http]
-    traceLogConsole = true
+    traceLogConsole = false
   {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}}
     [ballerina.observe]
     metricsEnabled=true
     metricsReporter="prometheus"
     [ballerinax.prometheus]
     port=18008
-  {{if .Values.wso2.apk.metrics.configdeployerhost}}
-    host={{.Values.wso2.apk.metrics.configdeployerhost}}
+  {{if .Values.wso2.apk.metrics.configDSBalPort}}
+    host="{{ .Values.wso2.apk.metrics.configDSBalPort}}"
   {{else}} 
-    host="127.0.0.1"
+    host="0.0.0.0"
   {{end}} 
   {{end}} 
     [wso2.config_deployer_service.keyStores.tls]

diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml
@@ -42,6 +42,12 @@ spec:
           ports:
             - containerPort: 9443
               protocol: "TCP"
+            {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }}
+            - containerPort: 18008
+              protocol: "TCP"
+            - containerPort: 18010
+              protocol: "TCP"
+            {{ end }}
           readinessProbe:
             httpGet:
               path: /health
@@ -68,6 +74,9 @@ spec:
             {{- else }}
               subPath: tls.key
             {{- end }}
+            - name: prometheus-jmx-config-volume
+              mountPath: /tmp/metrics/prometheus-jmx-config-configds.yml
+              subPath: prometheus-jmx-config-configds.yml
             - name: config-ds-tls-volume
               mountPath: /home/wso2apk/config-deployer/security/config.pem
             {{- if and .Values.wso2.apk.dp.configdeployer.configs .Values.wso2.apk.dp.configdeployer.configs.tls }}
@@ -105,6 +114,9 @@ spec:
         - name: config-toml-volume
           configMap:
             name: {{ template "apk-helm.resource.prefix" . }}-config-ds-configmap
+        - name: prometheus-jmx-config-volume
+          configMap: 
+            name: prometheus-jmx-config-configds
         - name: config-ds-tls-volume
           secret:
        {{ if and .Values.wso2.apk.dp.configdeployer.configs .Values.wso2.apk.dp.configdeployer.configs.tls }}

diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml
@@ -0,0 +1,47 @@
+# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved.
+#
+# WSO2 LLC. licenses this file to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-jmx-config-configds
+data:
+  prometheus-jmx-config-configds.yml: |
+    lowercaseOutputName: true
+    lowercaseOutputLabelNames: true
+    rules:
+      # WSO2 APK related metrics
+      - pattern: 'org.wso2.apk.config<type=ExtAuthMetrics><>total_request_count: (.*)'
+        name: org_wso2_apk_config_request_count_total
+        help: "WSO2 APK config total request count."
+        attrNameSnakeCase: true
+        type: COUNTER
+      - pattern: 'org.wso2.apk.config<type=ExtAuthMetrics><>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis): (.*)'
+        name: org_wso2_apk_config_$1
+        help: "WSO2 APK config $1."
+        attrNameSnakeCase: true
+        type: GAUGE
+      - pattern: 'org.wso2.apk.config<type=ThreadPoolConfig><>(\w+): (.*)'
+        name: org_wso2_apk_config_thread_pool_$1
+        help: "WSO2 APK config thread pool $1."
+        attrNameSnakeCase: true
+        type: GAUGE
+      # OS related metrics
+      - pattern: 'java.lang<type=OperatingSystem><>(\w+): (.*)'
+        name: os_$1
+        help: Operating System $1
+        attrNameSnakeCase: true
+        type: GAUGE
diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml
@@ -28,4 +28,12 @@ spec:
     - name: servlet-https
       protocol: TCP
       port: 9443
+    {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }}
+    - name: prometheus-config-ds-port-lang
+      protocol: TCP
+      port: 18008
+    - name: prometheus-config-ds-port-system
+      protocol: TCP
+      port: 18010
+    {{ end }}
 {{- end -}}
diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml
@@ -242,8 +242,9 @@ wso2:
             #     secretName: "router-cert"
             #     certKeyFilename: ""
             #     certFilename: ""
-  metrics:
-    enabled: true
+    metrics:
+      enabled: true
+      # configDSBalPort: 0.0.0.0
 
 idp:
   enabled: true

diff --git a/runtime/config-deployer-service/ballerina/APIClient.bal b/runtime/config-deployer-service/ballerina/APIClient.bal
@@ -27,10 +27,9 @@ import ballerina/uuid;
 import ballerina/crypto;
 import config_deployer_service.java.io as javaio;
 import wso2/apk_common_lib as commons;
-
-#
+import ballerinax/prometheus as _;
 public class APIClient {
-
+    
     # This function used to convert APKInternalAPI model to APKConf.
     #
     # + api - APKInternalAPI model

diff --git a/runtime/config-deployer-service/ballerina/Ballerina.toml b/runtime/config-deployer-service/ballerina/Ballerina.toml
@@ -5,7 +5,7 @@ version = "1.1.0-SNAPSHOT"
 distribution = "2201.8.0"
 
 [build-options]
-observabilityIncluded = false
+observabilityIncluded = true
 
 [[dependency]]
 org = "wso2"
@@ -49,6 +49,12 @@ groupId = "io.swagger"
 artifactId = "swagger-core"
 version = "1.6.10"
 
+# transitive dependency of io.prometheus.jmx:jmx_prometheus_javaagent:0.20.0
+[[platform.java11.dependency]]
+groupId = "io.prometheus.jmx"
+artifactId = "jmx_prometheus_javaagent"
+version = "0.20.0"
+
 # transitive dependency of io.swagger.parser.v3:swagger-parser:2.1.13
 [[platform.java11.dependency]]
 groupId = "io.swagger"

diff --git a/runtime/config-deployer-service/ballerina/Dependencies.toml b/runtime/config-deployer-service/ballerina/Dependencies.toml
@@ -365,6 +365,18 @@ modules = [
 	{org = "ballerina", packageName = "uuid", moduleName = "uuid"}
 ]
 
+[[package]]
+org = "ballerinai"
+name = "observe"
+version = "0.0.0"
+dependencies = [
+	{org = "ballerina", name = "jballerina.java"},
+	{org = "ballerina", name = "observe"}
+]
+modules = [
+	{org = "ballerinai", packageName = "observe", moduleName = "observe"}
+]
+
 [[package]]
 org = "ballerinax"
 name = "postgresql"
@@ -377,6 +389,21 @@ dependencies = [
 	{org = "ballerina", name = "time"}
 ]
 
+[[package]]
+org = "ballerinax"
+name = "prometheus"
+version = "0.2.0"
+dependencies = [
+	{org = "ballerina", name = "http"},
+	{org = "ballerina", name = "io"},
+	{org = "ballerina", name = "lang.string"},
+	{org = "ballerina", name = "observe"},
+	{org = "ballerina", name = "regex"}
+]
+modules = [
+	{org = "ballerinax", packageName = "prometheus", moduleName = "prometheus"}
+]
+
 [[package]]
 org = "wso2"
 name = "apk_common_lib"
@@ -419,6 +446,8 @@ dependencies = [
 	{org = "ballerina", name = "test"},
 	{org = "ballerina", name = "url"},
 	{org = "ballerina", name = "uuid"},
+	{org = "ballerinai", name = "observe"},
+	{org = "ballerinax", name = "prometheus"},
 	{org = "wso2", name = "apk_common_lib"}
 ]
 modules = [
@@ -431,6 +460,7 @@ modules = [
 	{org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.model"},
 	{org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config"},
 	{org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config.api"},
+	{org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config.definitions"},
 	{org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config.model"},
 	{org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.partitionClient"}
 ]

diff --git a/runtime/config-deployer-service/docker/Dockerfile b/runtime/config-deployer-service/docker/Dockerfile
@@ -53,6 +53,8 @@ RUN groupadd --system -g ${USER_GROUP_ID} ${USER_GROUP} \
 
 ADD config-deployer ${USER_HOME}/config-deployer
 COPY docker-entrypoint.sh ${USER_HOME}
+COPY /lib/jmx_prometheus_javaagent-0.20.0.jar /home/${USER}/lib/
+
 RUN chown -R ${USER} ${USER_HOME}/config-deployer \
     && chown ${USER} /home/${USER}/docker-entrypoint.sh \
     && chgrp -R 0 ${USER_HOME} \

diff --git a/runtime/config-deployer-service/docker/config-deployer/config.sh b/runtime/config-deployer-service/docker/config-deployer/config.sh
@@ -98,9 +98,10 @@ fi
 echo "Using Java memory options: $JVM_MEM_OPTS"
 
 $JAVACMD \
-    $JVM_MEM_OPTS \
-    $JAVA_OPTS \
-    -classpath "$CLASSPATH" \
-    -Djava.io.tmpdir="$RUNTIME_HOME/tmp" \
-    -jar config_deployer_service.jar $*
-    status=$?
+  $JVM_MEM_OPTS \
+  $JAVA_OPTS \
+  -classpath "$CLASSPATH" \
+  -Djava.io.tmpdir="$RUNTIME_HOME/tmp" \
+  -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18010:/tmp/metrics/prometheus-jmx-config-configds.yml \
+  -jar config_deployer_service.jar $*
+status=$?
diff --git a/runtime/config-deployer-service/java/build.gradle b/runtime/config-deployer-service/java/build.gradle
@@ -17,8 +17,6 @@
 
 apply from: "$projectDir/../../../common-gradle-scripts/java.gradle"
 
-
-
 dependencies {
     implementation libs.commons.logging
     implementation libs.gson
@@ -34,4 +32,35 @@ dependencies {
     implementation libs.handy.uri.templates
     implementation libs.json
     implementation libs.commons.lang
+    implementation libs.prometheus
+}
+
+tasks.register('copy_dependencies', Copy) {
+    dependsOn create_lib
+    from configurations.default
+    into "$projectDir/../docker/lib"
+    finalizedBy build
+}
+
+tasks.register('create_lib') {
+    dependsOn delete_lib
+    finalizedBy copy_dependencies
 }
+
+tasks.register('delete_lib', Delete) {
+    delete "$projectDir/lib"
+    finalizedBy create_lib
+}
+
+tasks.named('build').configure {
+    dependsOn copy_dependencies
+}
+
+configurations.implementation {
+    exclude group: 'jline'
+    exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+    exclude group: 'org.apache.logging'
+    exclude group: 'log4j'
+    exclude group: 'org.ops4j.pax.logging'
+    exclude group: 'com.google.guava'
+}