From 8f84267f0b0d44fe89f593f6f0f9e85f88ae515a Mon Sep 17 00:00:00 2001 From: CrowleyRajapakse Date: Thu, 18 Jul 2024 10:10:22 +0530 Subject: [PATCH] initial impl for business analytics for prometheus metrics --- .../analytics/publisher/jmx/JMXUtils.java | 34 ++ .../publisher/jmx/MBeanManagementFactory.java | 42 +++ .../publisher/jmx/MBeanRegistrator.java | 137 ++++++++ .../jmx/api/ExtAuthMetricsMXBean.java | 95 ++++++ .../publisher/jmx/impl/ExtAuthMetrics.java | 237 ++++++++++++++ .../reporter/GenericInputValidator.java | 72 +++++ .../reporter/MetricReporterFactory.java | 3 + .../publisher/reporter/MetricSchema.java | 2 + .../prometheus/APIInvocationEvent.java | 298 ++++++++++++++++++ .../prometheus/PrometheusCounterMetric.java | 156 +++++++++ .../PrometheusMetricEventBuilder.java | 125 ++++++++ .../prometheus/PrometheusMetricReporter.java | 56 ++++ .../analytics/publisher/util/Constants.java | 1 + .../apk/enforcer/config/ConfigHolder.java | 2 + .../apk/enforcer/constants/Constants.java | 1 + .../metrics/prometheus-jmx-config.yml | 45 +++ .../gateway-runtime-deployment.yaml | 6 +- .../prometheus-jmx-configmap.yaml | 26 +- helm-charts/values.yaml | 42 ++- 19 files changed, 1340 insertions(+), 40 deletions(-) create mode 100644 gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/JMXUtils.java create mode 100644 gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/MBeanManagementFactory.java create mode 100644 gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/MBeanRegistrator.java create mode 100644 gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/api/ExtAuthMetricsMXBean.java create mode 100644 gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/impl/ExtAuthMetrics.java create mode 100644 gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/APIInvocationEvent.java create mode 100644 gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusCounterMetric.java create mode 100644 gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusMetricEventBuilder.java create mode 100644 gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusMetricReporter.java diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/JMXUtils.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/JMXUtils.java new file mode 100644 index 0000000000..e7d15f2b67 --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/JMXUtils.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wso2.apk.enforcer.analytics.publisher.jmx; + +/** + * JMX Utilities + */ +public class JMXUtils { + + private static final String APK_JMX_METRICS_ENABLE = "apk.jmx.metrics.enabled"; + + /** + * Returns true if jmx metrics enabled as a system property, otherwise false. + * + * @return boolean + */ + public static boolean isJMXMetricsEnabled() { + return Boolean.getBoolean(APK_JMX_METRICS_ENABLE); + } +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/MBeanManagementFactory.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/MBeanManagementFactory.java new file mode 100644 index 0000000000..5dd41005f5 --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/MBeanManagementFactory.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.wso2.apk.enforcer.analytics.publisher.jmx; + +import javax.management.MBeanServer; +import javax.management.MBeanServerFactory; + +/** + * The ManagementFactory class is a factory class for getting managed beans for + * the Enforcer. + */ +public class MBeanManagementFactory { + + /* + * If one already exists, it will return that else it will create a new one and + * return. + * + * @return A MBeanServer instance. + */ + public static MBeanServer getMBeanServer() { + MBeanServer mBeanServer; + if (MBeanServerFactory.findMBeanServer(null).size() > 0) { + mBeanServer = MBeanServerFactory.findMBeanServer(null).get(0); + } else { + mBeanServer = MBeanServerFactory.createMBeanServer(); + } + return mBeanServer; + } +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/MBeanRegistrator.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/MBeanRegistrator.java new file mode 100644 index 0000000000..2ff04b824a --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/MBeanRegistrator.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2022, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.wso2.apk.enforcer.analytics.publisher.jmx; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.wso2.apk.enforcer.analytics.publisher.jmx.impl.ExtAuthMetrics; +import org.wso2.apk.enforcer.analytics.publisher.reporter.prometheus.APIInvocationEvent; + +import javax.management.*; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * The class which is responsible for registering MBeans. + */ +public class MBeanRegistrator { + private static final Log logger = LogFactory.getLog(MBeanRegistrator.class); + private static List mBeans = new ArrayList<>(); + + private static final String SERVER_PACKAGE = "org.wso2.apk.enforcer.analytics"; + + private MBeanRegistrator() { + } + + /** + * Registers an object as an MBean with the MBean server. + * + * @param mBeanInstance - The MBean to be registered as an MBean. + */ + public static void registerMBean(Object mBeanInstance, APIInvocationEvent event) throws RuntimeException, UnsupportedEncodingException { + + if (JMXUtils.isJMXMetricsEnabled()) { + String className = mBeanInstance.getClass().getName(); + if (className.indexOf('.') != -1) { + className = className.substring(className.lastIndexOf('.') + 1); + } + + ExtAuthMetrics extAuthMetrics = (ExtAuthMetrics) mBeanInstance; + logger.info("extAuthMetrics Object data: " + extAuthMetrics.toString()); + //logger.info("extAuthMetrics Object api name: " + extAuthMetrics.getApiName()); + + String objectName = String.format( + "%s:type=%s,apiName=%s,apiContext=%s,proxyResponseCode=%d," + + "destination=%s,apiCreatorTenantDomain=%s,platform=%s," + + "organizationId=%s,apiMethod=%s,apiVersion=%s,environmentId=%s," + + "gatewayType=%s,apiCreator=%s,responseCacheHit=%b,backendLatency=%d," + + "correlationId=%s,requestMediationLatency=%d,keyType=%s,apiId=%s," + + "applicationName=%s,targetResponseCode=%d,applicationOwner=%s," + + "userAgent=%s,userName=%s,apiResourceTemplate=%s,regionId=%s,responseLatency=%d," + + "responseMediationLatency=%d,userIp=%s,applicationId=%s,apiType=%s,xOriginalGwUrl=%s", + SERVER_PACKAGE, + className, + event.getApiName(), + event.getApiContext(), + event.getProxyResponseCode(), + URLEncoder.encode(event.getDestination(), StandardCharsets.UTF_8.toString()), + event.getApiCreatorTenantDomain(), + event.getPlatform(), + event.getOrganizationId(), + event.getApiMethod(), + event.getApiVersion(), + event.getEnvironmentId(), + event.getGatewayType(), + event.getApiCreator(), + event.isResponseCacheHit(), + event.getBackendLatency(), + event.getCorrelationId(), + event.getRequestMediationLatency(), + event.getKeyType(), + event.getApiId(), + "Resident Key Manager", + event.getTargetResponseCode(), + event.getApplicationOwner(), + event.getUserAgent(), + event.getUserName(), + event.getApiResourceTemplate(), + event.getRegionId(), + event.getResponseLatency(), + event.getResponseMediationLatency(), + event.getUserIp(), + event.getApplicationId(), + event.getApiType(), + URLEncoder.encode(event.getProperties().get("x-original-gw-url"), StandardCharsets.UTF_8.toString()) + ); + logger.info("Registering MBean with object name: " + objectName); + try { + MBeanServer mBeanServer = MBeanManagementFactory.getMBeanServer(); + Set set = mBeanServer.queryNames(new ObjectName(objectName), null); + if (set.isEmpty()) { + try { + ObjectName name = new ObjectName(objectName); + mBeanServer.registerMBean(mBeanInstance, name); + mBeans.add(name); + logger.info("MBean registered successfully with object name: " + name); + logger.info("Mbeans: " + mBeans); + } catch (InstanceAlreadyExistsException e) { + String msg = "MBean " + objectName + " already exists"; + logger.error(msg, e); + throw new RuntimeException(msg, e); + } catch (MBeanRegistrationException | NotCompliantMBeanException e) { + String msg = "Execption when registering MBean"; + logger.error(msg, e); + throw new RuntimeException(msg, e); + } + } else { + String msg = "MBean " + objectName + " already exists"; + logger.error(msg); + throw new RuntimeException(msg); + } + } catch (MalformedObjectNameException e) { + String msg = "Could not register " + mBeanInstance.getClass() + " MBean"; + logger.error(msg, e); + throw new RuntimeException(msg, e); + } + } else { + logger.debug("JMX Metrics should be enabled to register MBean instance: {}"); + } + } +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/api/ExtAuthMetricsMXBean.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/api/ExtAuthMetricsMXBean.java new file mode 100644 index 0000000000..895f62794a --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/api/ExtAuthMetricsMXBean.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wso2.apk.enforcer.analytics.publisher.jmx.api; + +/** + * MBean API for ExtAuth Service metrics. + */ +public interface ExtAuthMetricsMXBean { + + /** + * Getter for total request count. + * + * @return long + */ + public long getTotalRequestCount(); + + /** + * Getter for average response time in milli seconds. + * + * @return double + */ + public double getAverageResponseTimeMillis(); + + /** + * Getter for maximum response time in milliseconds. + * + * @return double + */ + public double getMaxResponseTimeMillis(); + + /** + * Getter for mimnimum response time in milliseconds. + * + * @return double + */ + public double getMinResponseTimeMillis(); + + /** + * Resets all the metrics to their initial values. + */ + public void resetExtAuthMetrics(); + + /** + * Get request count in last five minutes window + * + * @return long + */ + public long getRequestCountInLastFiveMinuteWindow(); + + /** + * Get the start time of request count window + * + * @return + */ + public long getRequestCountWindowStartTimeMillis(); + + /** + * Get the number of token issuers + * + * @return + */ + public int getTokenIssuerCount(); + + /** + * Get the number of subscriptions + * + * @return + */ + public int getSubscriptionCount(); + + public int getTotalRequests(); + public int getPostRequests(); + public int getGetRequests(); + public String getResourcePaths(); + public String getApiName(); + public String getApplicationId(); + public int getApiMessages(); + public int getSuccessRequests(); + public int getFailureRequests(); + +} \ No newline at end of file diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/impl/ExtAuthMetrics.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/impl/ExtAuthMetrics.java new file mode 100644 index 0000000000..61ca40898a --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/jmx/impl/ExtAuthMetrics.java @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wso2.apk.enforcer.analytics.publisher.jmx.impl; + +import org.wso2.apk.enforcer.analytics.publisher.jmx.MBeanRegistrator; +import org.wso2.apk.enforcer.analytics.publisher.jmx.api.ExtAuthMetricsMXBean; +import org.wso2.apk.enforcer.analytics.publisher.reporter.prometheus.APIInvocationEvent; + +import java.io.UnsupportedEncodingException; +import java.util.Timer; +import java.util.TimerTask; + +/** + * Singleton MBean for ExtAuth Service metrics. + */ +public class ExtAuthMetrics extends TimerTask implements ExtAuthMetricsMXBean { + + private static final long REQUEST_COUNT_INTERVAL_MILLIS = 5 * 60 * 1000; + private static ExtAuthMetrics extAuthMetricsMBean = null; + private long requestCountInLastFiveMinuteWindow = 0; + private int tokenIssuerCount = 0; + private long requestCountWindowStartTimeMillis = System.currentTimeMillis(); + private long totalRequestCount = 0; + private int subscriptionCount = 0; + private double averageResponseTimeMillis = 0; + private double maxResponseTimeMillis = Double.MIN_VALUE; + private double minResponseTimeMillis = Double.MAX_VALUE; + + private int apiMessages = 0; + private int totalRequests = 0; + private int postRequests = 0; + private int getRequests = 0; + private String resourcePaths = ""; + private String apiName = "sample1"; + private String applicationId = ""; + private int successRequests = 0; + private int failureRequests = 0; + +// private ExtAuthMetrics() { +// MBeanRegistrator.registerMBean(this); +// } + + private ExtAuthMetrics(APIInvocationEvent event) { + //getRequests++; + //this.apiName = apiName; + try { + MBeanRegistrator.registerMBean(this, event); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + +/** + * Getter for the Singleton ExtAuthMetrics instance. + * + * @return ExtAuthMetrics + */ +public static ExtAuthMetrics getInstance(APIInvocationEvent event) { + if (extAuthMetricsMBean == null) { + synchronized (ExtAuthMetrics.class) { + if (extAuthMetricsMBean == null) { + Timer timer = new Timer(); + extAuthMetricsMBean = new ExtAuthMetrics(event); + extAuthMetricsMBean.requestCountWindowStartTimeMillis = System.currentTimeMillis(); + timer.schedule(extAuthMetricsMBean, REQUEST_COUNT_INTERVAL_MILLIS, REQUEST_COUNT_INTERVAL_MILLIS); + } + } + } + return extAuthMetricsMBean; +} + + @Override + public long getTotalRequestCount() { + return totalRequestCount; + }; + + @Override + public double getAverageResponseTimeMillis() { + return averageResponseTimeMillis; + }; + + @Override + public double getMaxResponseTimeMillis() { + return maxResponseTimeMillis; + }; + + @Override + public double getMinResponseTimeMillis() { + return minResponseTimeMillis; + }; + + public synchronized void recordMetric(long responseTimeMillis) { + this.requestCountInLastFiveMinuteWindow += 1; + this.totalRequestCount += 1; + this.averageResponseTimeMillis = this.averageResponseTimeMillis + + (responseTimeMillis - this.averageResponseTimeMillis) / totalRequestCount; + this.minResponseTimeMillis = Math.min(this.minResponseTimeMillis, responseTimeMillis); + this.maxResponseTimeMillis = Math.max(this.maxResponseTimeMillis, responseTimeMillis); + } + + public synchronized void recordJWTIssuerMetrics(int jwtIssuers) { + this.tokenIssuerCount = jwtIssuers; + } + + public synchronized void recordSubscriptionMetrics(int subscriptionCount) { + this.subscriptionCount = subscriptionCount; + } + + @Override + public synchronized void resetExtAuthMetrics() { + this.totalRequestCount = 0; + this.averageResponseTimeMillis = 0; + this.maxResponseTimeMillis = Double.MIN_VALUE; + this.minResponseTimeMillis = Double.MAX_VALUE; + } + + @Override + public synchronized void run() { + requestCountWindowStartTimeMillis = System.currentTimeMillis(); + requestCountInLastFiveMinuteWindow = 0; + } + + @Override + public long getRequestCountInLastFiveMinuteWindow() { + return requestCountInLastFiveMinuteWindow; + } + + @Override + public long getRequestCountWindowStartTimeMillis() { + return requestCountWindowStartTimeMillis; + } + + @Override + public int getTokenIssuerCount() { + return tokenIssuerCount; + } + + @Override + public int getSubscriptionCount() { + return subscriptionCount; + } + + public void recordApiMessages() { + apiMessages++;; + } + + @Override + public int getApiMessages() { + return apiMessages; + } + + @Override + public int getTotalRequests() { + return totalRequests; + } + + @Override + public int getPostRequests() { + return postRequests; + } + + @Override + public int getGetRequests() { + return getRequests; + } + + @Override + public String getResourcePaths() { + return resourcePaths; + } + + public void incrementTotalRequests() { + totalRequests++; + } + + public void incrementPostRequests() { + postRequests++; + } + + public void incrementGetRequests() { + getRequests++; + } + + public void incrementResourcePath(String resourcePath) { + resourcePaths = resourcePath; + } + + public void incrementApiName(String apiName) { + this.apiName = apiName; + } + + @Override + public String getApiName() { + return apiName; + } + + public void incrementApplicationId(String applicationId) { + this.applicationId = applicationId; + } + + @Override + public String getApplicationId() { + return applicationId; + } + + public void incrementSuccessRequests() { + successRequests++; + } + + public void incrementFailureRequests() { + failureRequests++; + } + + @Override + public int getSuccessRequests() { + return successRequests; + } + + @Override + public int getFailureRequests() { + return failureRequests; + } +} \ No newline at end of file diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/GenericInputValidator.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/GenericInputValidator.java index 90ddb5ce62..8545026c02 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/GenericInputValidator.java +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/GenericInputValidator.java @@ -250,6 +250,74 @@ public class GenericInputValidator { new AbstractMap.SimpleImmutableEntry<>(USER_AGENT_HEADER, String.class), new AbstractMap.SimpleImmutableEntry<>(PROPERTIES, LinkedHashMap.class)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + private static final Map prometheusResponseEventSchema = Stream.of( + new AbstractMap.SimpleImmutableEntry<>(REQUEST_TIMESTAMP, String.class), + new AbstractMap.SimpleImmutableEntry<>(CORRELATION_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(KEY_TYPE, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_TYPE, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_NAME, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_VERSION, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_CREATION, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_METHOD, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_RESOURCE_TEMPLATE, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_CREATOR_TENANT_DOMAIN, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_CONTEXT, String.class), + new AbstractMap.SimpleImmutableEntry<>(DESTINATION, String.class), + new AbstractMap.SimpleImmutableEntry<>(APPLICATION_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(APPLICATION_NAME, String.class), + new AbstractMap.SimpleImmutableEntry<>(APPLICATION_OWNER, String.class), + new AbstractMap.SimpleImmutableEntry<>(REGION_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(ORGANIZATION_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(ENVIRONMENT_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(GATEWAY_TYPE, String.class), + new AbstractMap.SimpleImmutableEntry<>(USER_AGENT_HEADER, String.class), + new AbstractMap.SimpleImmutableEntry<>(USER_NAME, String.class), + new AbstractMap.SimpleImmutableEntry<>(PROXY_RESPONSE_CODE, Integer.class), + new AbstractMap.SimpleImmutableEntry<>(TARGET_RESPONSE_CODE, Integer.class), + new AbstractMap.SimpleImmutableEntry<>(RESPONSE_CACHE_HIT, Boolean.class), + new AbstractMap.SimpleImmutableEntry<>(RESPONSE_LATENCY, Long.class), + new AbstractMap.SimpleImmutableEntry<>(BACKEND_LATENCY, Long.class), + new AbstractMap.SimpleImmutableEntry<>(REQUEST_MEDIATION_LATENCY, Long.class), + new AbstractMap.SimpleImmutableEntry<>(RESPONSE_MEDIATION_LATENCY, Long.class), + new AbstractMap.SimpleImmutableEntry<>(USER_IP, String.class), + new AbstractMap.SimpleImmutableEntry<>(PROPERTIES, LinkedHashMap.class)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + private static final Map prometheusFaultEventSchema = Stream.of( + new AbstractMap.SimpleImmutableEntry<>(REQUEST_TIMESTAMP, String.class), + new AbstractMap.SimpleImmutableEntry<>(CORRELATION_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(KEY_TYPE, String.class), + new AbstractMap.SimpleImmutableEntry<>(ERROR_TYPE, String.class), + new AbstractMap.SimpleImmutableEntry<>(ERROR_CODE, Integer.class), + new AbstractMap.SimpleImmutableEntry<>(ERROR_MESSAGE, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_TYPE, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_NAME, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_VERSION, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_CREATION, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_METHOD, String.class), + new AbstractMap.SimpleImmutableEntry<>(API_CREATOR_TENANT_DOMAIN, String.class), + new AbstractMap.SimpleImmutableEntry<>(APPLICATION_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(APPLICATION_NAME, String.class), + new AbstractMap.SimpleImmutableEntry<>(APPLICATION_OWNER, String.class), + new AbstractMap.SimpleImmutableEntry<>(REGION_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(ORGANIZATION_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(ENVIRONMENT_ID, String.class), + new AbstractMap.SimpleImmutableEntry<>(GATEWAY_TYPE, String.class), + new AbstractMap.SimpleImmutableEntry<>(USER_AGENT_HEADER, String.class), + new AbstractMap.SimpleImmutableEntry<>(USER_NAME, String.class), + new AbstractMap.SimpleImmutableEntry<>(PROXY_RESPONSE_CODE, Integer.class), + new AbstractMap.SimpleImmutableEntry<>(TARGET_RESPONSE_CODE, Integer.class), + new AbstractMap.SimpleImmutableEntry<>(RESPONSE_CACHE_HIT, Boolean.class), + new AbstractMap.SimpleImmutableEntry<>(RESPONSE_LATENCY, Long.class), + new AbstractMap.SimpleImmutableEntry<>(BACKEND_LATENCY, Long.class), + new AbstractMap.SimpleImmutableEntry<>(REQUEST_MEDIATION_LATENCY, Long.class), + new AbstractMap.SimpleImmutableEntry<>(RESPONSE_MEDIATION_LATENCY, Long.class), + new AbstractMap.SimpleImmutableEntry<>(USER_IP, String.class), + new AbstractMap.SimpleImmutableEntry<>(PROPERTIES, LinkedHashMap.class)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); private static final List configProperties = new ArrayList<>(); private GenericInputValidator() { @@ -276,6 +344,10 @@ public Map getEventProperties(MetricSchema schema) { return elkResponseEventSchema; case ELK_ERROR: return elkFaultEventSchema; + case PROMETHEUS_RESPONSE: + return prometheusResponseEventSchema; + case PROMETHEUS_ERROR: + return prometheusFaultEventSchema; default: return new HashMap<>(); } diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/MetricReporterFactory.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/MetricReporterFactory.java index ad9aed2ccd..2d0dae79a3 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/MetricReporterFactory.java +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/MetricReporterFactory.java @@ -24,6 +24,7 @@ import org.wso2.apk.enforcer.analytics.publisher.reporter.cloud.DefaultAnalyticsMetricReporter; import org.wso2.apk.enforcer.analytics.publisher.reporter.elk.ELKMetricReporter; import org.wso2.apk.enforcer.analytics.publisher.reporter.moesif.MoesifReporter; +import org.wso2.apk.enforcer.analytics.publisher.reporter.prometheus.PrometheusMetricReporter; import org.wso2.apk.enforcer.analytics.publisher.util.Constants; import java.lang.reflect.Constructor; @@ -125,6 +126,8 @@ public MetricReporter createMetricReporterFromType(String type, Map properties; + + // Getters and setters for all fields + public String getApiName() { + return apiName; + } + + public void setApiName(String apiName) { + this.apiName = apiName; + } + + public int getProxyResponseCode() { + return proxyResponseCode; + } + + public void setProxyResponseCode(int proxyResponseCode) { + this.proxyResponseCode = proxyResponseCode; + } + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public String getApiCreatorTenantDomain() { + return apiCreatorTenantDomain; + } + + public void setApiCreatorTenantDomain(String apiCreatorTenantDomain) { + this.apiCreatorTenantDomain = apiCreatorTenantDomain; + } + + public String getPlatform() { + return platform; + } + + public void setPlatform(String platform) { + this.platform = platform; + } + + public String getOrganizationId() { + return organizationId; + } + + public void setOrganizationId(String organizationId) { + this.organizationId = organizationId; + } + + public String getApiMethod() { + return apiMethod; + } + + public void setApiMethod(String apiMethod) { + this.apiMethod = apiMethod; + } + + public String getApiVersion() { + return apiVersion; + } + + public void setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + } + + public String getGatewayType() { + return gatewayType; + } + + public void setGatewayType(String gatewayType) { + this.gatewayType = gatewayType; + } + + public String getEnvironmentId() { + return environmentId; + } + + public void setEnvironmentId(String environmentId) { + this.environmentId = environmentId; + } + + public String getApiCreator() { + return apiCreator; + } + + public void setApiCreator(String apiCreator) { + this.apiCreator = apiCreator; + } + + public boolean isResponseCacheHit() { + return responseCacheHit; + } + + public void setResponseCacheHit(boolean responseCacheHit) { + this.responseCacheHit = responseCacheHit; + } + + public int getBackendLatency() { + return backendLatency; + } + + public void setBackendLatency(int backendLatency) { + this.backendLatency = backendLatency; + } + + public String getCorrelationId() { + return correlationId; + } + + public void setCorrelationId(String correlationId) { + this.correlationId = correlationId; + } + + public int getRequestMediationLatency() { + return requestMediationLatency; + } + + public void setRequestMediationLatency(int requestMediationLatency) { + this.requestMediationLatency = requestMediationLatency; + } + + public String getKeyType() { + return keyType; + } + + public void setKeyType(String keyType) { + this.keyType = keyType; + } + + public String getApiId() { + return apiId; + } + + public void setApiId(String apiId) { + this.apiId = apiId; + } + + public String getApplicationName() { + return applicationName; + } + + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + + public int getTargetResponseCode() { + return targetResponseCode; + } + + public void setTargetResponseCode(int targetResponseCode) { + this.targetResponseCode = targetResponseCode; + } + + public String getRequestTimestamp() { + return requestTimestamp; + } + + public void setRequestTimestamp(String requestTimestamp) { + this.requestTimestamp = requestTimestamp; + } + + public String getApplicationOwner() { + return applicationOwner; + } + + public void setApplicationOwner(String applicationOwner) { + this.applicationOwner = applicationOwner; + } + + public String getUserAgent() { + return userAgent; + } + + public void setUserAgent(String userAgent) { + this.userAgent = userAgent; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getApiResourceTemplate() { + return apiResourceTemplate; + } + + public void setApiResourceTemplate(String apiResourceTemplate) { + this.apiResourceTemplate = apiResourceTemplate; + } + + public String getRegionId() { + return regionId; + } + + public void setRegionId(String regionId) { + this.regionId = regionId; + } + + public int getResponseLatency() { + return responseLatency; + } + + public void setResponseLatency(int responseLatency) { + this.responseLatency = responseLatency; + } + + public int getResponseMediationLatency() { + return responseMediationLatency; + } + + public void setResponseMediationLatency(int responseMediationLatency) { + this.responseMediationLatency = responseMediationLatency; + } + + public String getUserIp() { + return userIp; + } + + public void setUserIp(String userIp) { + this.userIp = userIp; + } + + public String getApiContext() { + return apiContext; + } + + public void setApiContext(String apiContext) { + this.apiContext = apiContext; + } + + public String getApplicationId() { + return applicationId; + } + + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + + public String getApiType() { + return apiType; + } + + public void setApiType(String apiType) { + this.apiType = apiType; + } + + public Map getProperties() { + return properties; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + + public APIInvocationEvent() { + } +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusCounterMetric.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusCounterMetric.java new file mode 100644 index 0000000000..aa028ee600 --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusCounterMetric.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.apk.enforcer.analytics.publisher.reporter.prometheus; + +import com.google.gson.Gson; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.wso2.apk.enforcer.analytics.publisher.exception.MetricReportingException; +import org.wso2.apk.enforcer.analytics.publisher.jmx.JMXUtils; +import org.wso2.apk.enforcer.analytics.publisher.jmx.impl.ExtAuthMetrics; +import org.wso2.apk.enforcer.analytics.publisher.reporter.CounterMetric; +import org.wso2.apk.enforcer.analytics.publisher.reporter.GenericInputValidator; +import org.wso2.apk.enforcer.analytics.publisher.reporter.MetricEventBuilder; +import org.wso2.apk.enforcer.analytics.publisher.reporter.MetricSchema; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** + * Prometheus Counter Metrics class, This class can be used to send analytics event as a prometheus metric event. + */ +public class PrometheusCounterMetric implements CounterMetric { + private static final Log log = LogFactory.getLog(PrometheusCounterMetric.class); + private final String name; + private final Gson gson; + private MetricSchema schema; + + protected PrometheusCounterMetric(String name, MetricSchema schema) { + this.name = name; + this.gson = new Gson(); + this.schema = schema; + } + + @Override + public int incrementCount(MetricEventBuilder builder) throws MetricReportingException { + + Map event = builder.build(); + String jsonString = gson.toJson(event); + log.info("JSON String:"+jsonString); + // Escape the double quotes +// String escapedJsonString = jsonString.replace("\"", "\\\""); + + // URL encode the JSON string + String encodedJsonString = null; + try { + encodedJsonString = URLEncoder.encode(jsonString, StandardCharsets.UTF_8.toString()); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + + log.info("JSON String: " + encodedJsonString); + APIInvocationEvent apiInvocationEvent = gson.fromJson(jsonString, APIInvocationEvent.class); + String apiName = apiInvocationEvent.getApiName(); + int proxyResponseCode = apiInvocationEvent.getProxyResponseCode(); + String destination = apiInvocationEvent.getDestination(); + String apiCreatorTenantDomain = apiInvocationEvent.getApiCreatorTenantDomain(); + String platform = apiInvocationEvent.getPlatform(); + String organizationId = apiInvocationEvent.getOrganizationId(); + String apiMethod = apiInvocationEvent.getApiMethod(); + String apiVersion = apiInvocationEvent.getApiVersion(); + String gatewayType = apiInvocationEvent.getGatewayType(); + String environmentId = apiInvocationEvent.getEnvironmentId(); + String apiCreator = apiInvocationEvent.getApiCreator(); + boolean responseCacheHit = apiInvocationEvent.isResponseCacheHit(); + int backendLatency = apiInvocationEvent.getBackendLatency(); + String correlationId = apiInvocationEvent.getCorrelationId(); + int requestMediationLatency = apiInvocationEvent.getRequestMediationLatency(); + String keyType = apiInvocationEvent.getKeyType(); + String apiId = apiInvocationEvent.getApiId(); + String applicationName = apiInvocationEvent.getApplicationName(); + int targetResponseCode = apiInvocationEvent.getTargetResponseCode(); + String requestTimestamp = apiInvocationEvent.getRequestTimestamp(); + String applicationOwner = apiInvocationEvent.getApplicationOwner(); + String userAgent = apiInvocationEvent.getUserAgent(); + String userName = apiInvocationEvent.getUserName(); + String apiResourceTemplate = apiInvocationEvent.getApiResourceTemplate(); + String regionId = apiInvocationEvent.getRegionId(); + int responseLatency = apiInvocationEvent.getResponseLatency(); + int responseMediationLatency = apiInvocationEvent.getResponseMediationLatency(); + String userIp = apiInvocationEvent.getUserIp(); + String apiContext = apiInvocationEvent.getApiContext(); + String applicationId = apiInvocationEvent.getApplicationId(); + String apiType = apiInvocationEvent.getApiType(); + Map properties = apiInvocationEvent.getProperties(); + + log.info("PrometheusMetrics: " + name.replaceAll("[\r\n]", "") + ", properties :" + + jsonString.replaceAll("[\r\n]", "")); + + String propertiesString = encodedJsonString; + log.info("Properties String:"+propertiesString); + if (JMXUtils.isJMXMetricsEnabled()) { + ExtAuthMetrics.getInstance(apiInvocationEvent).recordApiMessages(); + ExtAuthMetrics.getInstance(apiInvocationEvent).incrementTotalRequests(); + if (apiMethod.equalsIgnoreCase("POST")) { + ExtAuthMetrics.getInstance(apiInvocationEvent).incrementPostRequests(); + } else if (apiMethod.equalsIgnoreCase("GET")) { + ExtAuthMetrics.getInstance(apiInvocationEvent).incrementGetRequests(); + log.info("apiMethod: " + apiMethod); + } + log.info("Proxy Response Code: " + proxyResponseCode); + log.info("Target Response Code: " + targetResponseCode); + if (proxyResponseCode == 200 || proxyResponseCode == 201 || proxyResponseCode == 202) { + ExtAuthMetrics.getInstance(apiInvocationEvent).incrementSuccessRequests(); + } else { + ExtAuthMetrics.getInstance(apiInvocationEvent).incrementFailureRequests(); + } + ExtAuthMetrics.getInstance(apiInvocationEvent).incrementResourcePath(properties.get("x-original-gw-url")); + ExtAuthMetrics.getInstance(apiInvocationEvent).incrementApiName(apiName); + ExtAuthMetrics.getInstance(apiInvocationEvent).incrementApplicationId(applicationId); + log.info("apiName: " + apiName + ", applicationId: " + applicationId); + } + return 0; + } + + @Override + public String getName() { + return name; + } + + @Override + public MetricSchema getSchema() { + return schema; + } + + @Override + public MetricEventBuilder getEventBuilder() { + + switch (schema) { + case RESPONSE: + default: + return new PrometheusMetricEventBuilder( + GenericInputValidator.getInstance().getEventProperties(MetricSchema.PROMETHEUS_RESPONSE)); + case ERROR: + return new PrometheusMetricEventBuilder( + GenericInputValidator.getInstance().getEventProperties(MetricSchema.PROMETHEUS_ERROR)); + } + } +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusMetricEventBuilder.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusMetricEventBuilder.java new file mode 100644 index 0000000000..1b430edd80 --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusMetricEventBuilder.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.apk.enforcer.analytics.publisher.reporter.prometheus; + +import org.wso2.apk.enforcer.analytics.publisher.exception.MetricReportingException; +import org.wso2.apk.enforcer.analytics.publisher.reporter.AbstractMetricEventBuilder; +import org.wso2.apk.enforcer.analytics.publisher.reporter.GenericInputValidator; +import org.wso2.apk.enforcer.analytics.publisher.reporter.MetricEventBuilder; +import org.wso2.apk.enforcer.analytics.publisher.reporter.MetricSchema; +import org.wso2.apk.enforcer.analytics.publisher.util.Constants; +import org.wso2.apk.enforcer.analytics.publisher.util.EventMapAttributeFilter; +import org.wso2.apk.enforcer.analytics.publisher.util.UserAgentParser; +import ua_parser.Client; + +import java.util.HashMap; +import java.util.Map; + +/** + * Event builder for Prometheus Metric Reporter. + */ +public class PrometheusMetricEventBuilder extends AbstractMetricEventBuilder { + + protected Map requiredAttributes; + private Map eventMap; + private Boolean isBuilt = false; + + public PrometheusMetricEventBuilder() { + requiredAttributes = GenericInputValidator.getInstance().getEventProperties(MetricSchema.RESPONSE); + eventMap = new HashMap<>(); + } + + public PrometheusMetricEventBuilder(Map requiredAttributes) { + this.requiredAttributes = requiredAttributes; + eventMap = new HashMap<>(); + } + + @Override + protected Map buildEvent() { + if (!isBuilt) { + // util function to filter required attributes + eventMap = EventMapAttributeFilter.getInstance().filter(eventMap, requiredAttributes); + + // userAgent raw string is not required and removing + String userAgentHeader = (String) eventMap.remove(Constants.USER_AGENT_HEADER); + if (userAgentHeader != null) { + setUserAgentProperties(userAgentHeader); + } + isBuilt = true; + } + return eventMap; + } + + @Override + public boolean validate() throws MetricReportingException { + if (!isBuilt) { + Map propertyMap = (Map) eventMap.get(Constants.PROPERTIES); + if (propertyMap != null) { + copyDefaultPropertiesToRootLevel(propertyMap); + } + for (Map.Entry entry : requiredAttributes.entrySet()) { + Object attribute = eventMap.get(entry.getKey()); + if (attribute == null) { + throw new MetricReportingException(entry.getKey() + " is missing in metric data. This metric event " + + "will not be processed further."); + } else if (!attribute.getClass().equals(entry.getValue())) { + throw new MetricReportingException(entry.getKey() + " is expecting a " + entry.getValue() + " type " + + "attribute while attribute of type " + + attribute.getClass() + " is present."); + } + } + } + return true; + } + + @Override + public MetricEventBuilder addAttribute(String key, Object value) throws MetricReportingException { + eventMap.put(key, value); + return this; + } + + private void setUserAgentProperties(String userAgentHeader) { + String browser = null; + String platform = null; + Client client = UserAgentParser.getInstance().parseUserAgent(userAgentHeader); + if (client != null) { + browser = client.userAgent.family; + platform = client.os.family; + } + + if (browser == null || browser.isEmpty()) { + browser = Constants.UNKNOWN_VALUE; + } + if (platform == null || platform.isEmpty()) { + platform = Constants.UNKNOWN_VALUE; + } + eventMap.put(Constants.USER_AGENT, browser); + eventMap.put(Constants.PLATFORM, platform); + } + + private void copyDefaultPropertiesToRootLevel(Map properties) { + + if (properties.get(Constants.API_CONTEXT) != null) { + eventMap.put(Constants.API_CONTEXT, properties.get(Constants.API_CONTEXT)); + } + if (properties.get(Constants.USER_NAME) != null) { + eventMap.put(Constants.USER_NAME, properties.get(Constants.USER_NAME)); + } + } +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusMetricReporter.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusMetricReporter.java new file mode 100644 index 0000000000..4de2f2508b --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/reporter/prometheus/PrometheusMetricReporter.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.apk.enforcer.analytics.publisher.reporter.prometheus; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.wso2.apk.enforcer.analytics.publisher.exception.MetricCreationException; +import org.wso2.apk.enforcer.analytics.publisher.reporter.AbstractMetricReporter; +import org.wso2.apk.enforcer.analytics.publisher.reporter.CounterMetric; +import org.wso2.apk.enforcer.analytics.publisher.reporter.MetricSchema; +import org.wso2.apk.enforcer.analytics.publisher.reporter.TimerMetric; + +import java.util.Map; + +/** + * Prometheus Metric Reporter class. + */ +public class PrometheusMetricReporter extends AbstractMetricReporter { + private static final Logger log = LoggerFactory.getLogger(PrometheusMetricReporter.class); + + public PrometheusMetricReporter(Map properties) throws MetricCreationException { + super(properties); + log.info("LogMetricReporter successfully initialized"); + } + + @Override + protected void validateConfigProperties(Map properties) throws MetricCreationException { + //nothing to validate + } + + @Override + protected CounterMetric createCounter(String name, MetricSchema schema) { + return new PrometheusCounterMetric(name, schema); + } + + @Override + protected TimerMetric createTimer(String name) { + return null; + } +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/util/Constants.java b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/util/Constants.java index ae55fb1927..a6474ea325 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/util/Constants.java +++ b/gateway/enforcer/org.wso2.apk.enforcer.analytics.publishers/src/main/java/org/wso2/apk/enforcer/analytics/publisher/util/Constants.java @@ -87,6 +87,7 @@ public class Constants { public static final String DEFAULT_REPORTER = "default"; public static final String ELK_REPORTER = "elk"; public static final String MOESIF_REPORTER = "moesif"; + public static final String PROMETHEUS_REPORTER = "prometheus"; //EventHub Client retry options constants public static final int DEFAULT_MAX_RETRIES = 2; diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java index c10358f336..7cb816450f 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java @@ -444,6 +444,8 @@ private void populateAnalyticsConfig(Analytics analyticsConfig) { resolvedConfigMap.put(Constants.AUTH_URL_CONFIG_TOKEN, authToken); } else if (analyticsPublisher.getType().equalsIgnoreCase(Constants.MOESIF_ANALYTICS_PUBLISHER)){ resolvedConfigMap.put(Constants.MOESIF_TOKEN, moesifToken); + } else if (analyticsPublisher.getType().equalsIgnoreCase(Constants.PROMETHEUS_ANALYTICS_PUBLISHER)){ + //resolvedConfigMap.put(Constants.PROMETHEUS_ANALYTICS_PUBLISHER, Constants.PROMETHEUS_ANALYTICS_PUBLISHER); } analyticsDTO.addAnalyticsPublisherConfig(new AnalyticsPublisherConfigDTO(analyticsPublisher.getEnabled(), analyticsPublisher.getType(), resolvedConfigMap)); diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/Constants.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/Constants.java index ed2cdb5427..5eb203e931 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/Constants.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/Constants.java @@ -86,4 +86,5 @@ since new lines in different OSs differ (Linux: \n, Windows: \r\n) */ public static final String MOESIF_TOKEN = "moesifToken"; public static final String DEFAULT_ANALYTICS_PUBLISHER = "default"; public static final String MOESIF_ANALYTICS_PUBLISHER = "moesif"; + public static final String PROMETHEUS_ANALYTICS_PUBLISHER = "prometheus"; } diff --git a/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml b/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml index 5771c00273..a59fe95588 100644 --- a/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml +++ b/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml @@ -31,6 +31,11 @@ rules: help: "WSO2 APK enforcer $1." attrNameSnakeCase: true type: GAUGE + - pattern: 'org.wso2.apk.enforcer.analytics<>(subscription_count|api_messages): (.*)' + name: org_wso2_apk_enforcer_analytics_$1 + help: "WSO2 APK enforcer analytics $1." + attrNameSnakeCase: true + type: GAUGE - pattern: 'org.wso2.apk.enforcer<>(\w+): (.*)' name: org_wso2_apk_enforcer_thread_pool_$1 help: "WSO2 APK enforcer thread pool $1." @@ -42,3 +47,43 @@ rules: help: Operating System $1 attrNameSnakeCase: true type: GAUGE + - pattern: 'org.wso2.apk.enforcer.analytics<>(total_requests): (.*)' + name: "api_http_requests_total" + attrNameSnakeCase: true + labels: + method: "ALL" + - pattern: 'org.wso2.apk.enforcer.analytics<>(post_requests): (.*)' + name: "api_http_requests_total" + attrNameSnakeCase: true + labels: + method: "POST" + - pattern: 'org.wso2.apk.enforcer.analytics<>(get_requests): (.*)' + name: "api_http_requests_total" + attrNameSnakeCase: true + labels: + method: "GET" + - pattern: 'org.wso2.apk.enforcer.analytics<>(resource_paths): (.*)' + name: "api_http_requests_total" + attrNameSnakeCase: true + labels: + resource_paths: "$1" + - pattern: 'org.wso2.apk.enforcer.analytics<>(api_name): (.*)' + name: "api_http_requests_total" + attrNameSnakeCase: true + labels: + api_name: "$1" + - pattern: 'org.wso2.apk.enforcer.analytics<>(application_id): (.*)' + name: "api_http_requests_total" + attrNameSnakeCase: true + labels: + application_id: "$1" + - pattern: 'org.wso2.apk.enforcer.analytics<>(success_requests): (.*)' + name: "api_http_requests_total" + attrNameSnakeCase: true + labels: + request_status: "SUCCESS" + - pattern: 'org.wso2.apk.enforcer.analytics<>(failure_requests): (.*)' + name: "api_http_requests_total" + attrNameSnakeCase: true + labels: + request_status: "FAILURE" diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index 3d6fc60622..cae694a0aa 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -56,6 +56,10 @@ spec: protocol: "TCP" - containerPort: 18002 protocol: "TCP" + - containerPort: 11111 + protocol: "TCP" + - containerPort: 9999 + protocol: "TCP" {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} - containerPort: 18006 protocol: "TCP" @@ -96,7 +100,7 @@ spec: value: admin - name: JAVA_OPTS {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18006:/tmp/metrics/prometheus-jmx-config-enforcer.yml + value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=11111 -Dcom.sun.management.jmxremote.rmi.port=9999 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=0.0.0.0 -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18006:/tmp/metrics/prometheus-jmx-config-enforcer.yml {{- else }} value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 {{- end }} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml index b7d676e47c..3657412a30 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml @@ -22,28 +22,8 @@ metadata: data: prometheus-jmx-config-enforcer.yml: | lowercaseOutputName: true - lowercaseOutputLabelNames: true + lowercaseOutputLabelNames: false rules: # WSO2 APK related metrics - - pattern: 'org.wso2.apk.enforcer<>total_request_count: (.*)' - name: org_wso2_apk_enforcer_request_count_total - help: "WSO2 APK enforcer total request count." - attrNameSnakeCase: true - type: COUNTER - - pattern: 'org.wso2.apk.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis|token_issuer_count|subscription_count): (.*)' - name: org_wso2_apk_enforcer_$1 - help: "WSO2 APK enforcer $1." - attrNameSnakeCase: true - type: GAUGE - - pattern: 'org.wso2.apk.enforcer<>(\w+): (.*)' - name: org_wso2_apk_enforcer_thread_pool_$1 - help: "WSO2 APK enforcer thread pool $1." - attrNameSnakeCase: true - type: GAUGE - # OS related metrics - - pattern: 'java.lang<>(\w+): (.*)' - name: os_$1 - help: Operating System $1 - attrNameSnakeCase: true - type: GAUGE -{{- end -}} \ No newline at end of file + - pattern: ".*" + {{- end -}} \ No newline at end of file diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index 7007fe8125..e1ce5629bb 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -191,6 +191,16 @@ wso2: # certFilename: "" # certCAFilename: "" gatewayRuntime: + analytics: + enabled: true + publishers: + - enabled: true + type: "default" + secretName: choreo-analytics-secret + - enabled: true + type: "elk" + - enabled: true + type: "prometheus" deployment: replicas: 1 router: @@ -241,8 +251,8 @@ wso2: periodSeconds: 20 failureThreshold: 5 strategy: RollingUpdate - imagePullPolicy: Always - image: wso2/apk-enforcer:1.1.0 + imagePullPolicy: IfNotPresent + image: apk-enforcer:1.1.0-SNAPSHOT security: sslHostname: "enforcer" # logging: @@ -257,20 +267,20 @@ wso2: # hostnameVerifier: "AllowAll" metrics: - enabled: false - # configDSBalHost: 0.0.0.0 - # idpDSBalHost: 0.0.0.0 - # statsd: - # image: - # repository: prom/statsd-exporter - # tag: v0.26.0 - # imagePullPolicy: IfNotPresent - # resources: - # limits: - # memory: 128Mi - # requests: - # cpu: 0.1 - # memory: 64Mi + enabled: true + configDSBalHost: 0.0.0.0 + idpDSBalHost: 0.0.0.0 + statsd: + image: + repository: prom/statsd-exporter + tag: v0.26.0 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 128Mi + requests: + cpu: 0.1 + memory: 64Mi idp: enabled: true listener: