diff --git a/.pipelines/azure-pipeline-build.yml b/.pipelines/azure-pipeline-build.yml index ebee81b37..f3717fb0d 100644 --- a/.pipelines/azure-pipeline-build.yml +++ b/.pipelines/azure-pipeline-build.yml @@ -600,7 +600,7 @@ jobs: - task: HelmInstaller@1 displayName: 'Build: install Helm version' inputs: - helmVersionToInstall: latest + helmVersionToInstall: 3.12.3 - bash: | envsubst < $(Build.SourcesDirectory)/otelcollector/deploy/chart/prometheus-collector/Chart-template.yaml > $(Build.SourcesDirectory)/otelcollector/deploy/chart/prometheus-collector/Chart.yaml && envsubst < $(Build.SourcesDirectory)/otelcollector/deploy/chart/prometheus-collector/values-template.yaml > $(Build.SourcesDirectory)/otelcollector/deploy/chart/prometheus-collector/values.yaml @@ -643,7 +643,7 @@ jobs: - task: HelmInstaller@1 displayName: 'Build: install Helm version' inputs: - helmVersionToInstall: latest + helmVersionToInstall: 3.12.3 - bash: | export HELM_CHART_NAME=$ARC_HELM_CHART_NAME diff --git a/AddonBicepTemplate/README.md b/AddonBicepTemplate/README.md index 84c21dccf..85af60539 100644 --- a/AddonBicepTemplate/README.md +++ b/AddonBicepTemplate/README.md @@ -5,7 +5,7 @@ You can deploy the templates using a command like : In order to deploy community alerts and ci recommended alerts through template, deploy using command like: -```az deployment group create -g -n --template-file .\AzureMonitorAlertsProfileParameters.json --parameters .\AzureMonitorAlertsProfileParameters.json``` +```az deployment group create -g -n --template-file .\AzureMonitorAlertsProfile.bicep --parameters .\AzureMonitorAlertsProfileParameters.json``` **NOTE** diff --git a/internal/alerts/ci-dev-aks-mac-eus.json b/internal/alerts/ci-dev-aks-mac-eus.json deleted file mode 100644 index 3c91fa6a8..000000000 --- a/internal/alerts/ci-dev-aks-mac-eus.json +++ /dev/null @@ -1,459 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "name": "containerinsights_dev_mac_eus_alerts", - "type": "Microsoft.AlertsManagement/prometheusRuleGroups", - "apiVersion": "2023-03-01", - "location": "eastus", - "properties": { - "description": "rule group for cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-mac-eus in MAC: /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-eus-mac", - "scopes": [ - "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-eus-mac" - ], - "rules": [ - { - "alert": "Amd64 metric missing in cluster ci-dev-aks-mac-eus", - "expression": "absent(node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Amd64 metric missing in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = node in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = node in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kubelet in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kubelet\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kubelet in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = windows-exporter in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"windows-exporter\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"windows-exporter\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = windows-exporter in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-apiserver in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-apiserver\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-apiserver in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy-windows\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy-windows\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = cadvisor in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = cadvisor in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-dns in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-dns\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-dns in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-mac-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-mac-eus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-mac-eus\"}) )) by (container, pod) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "CPU usage greater than 90% for prometheus-collector on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 5 for prometheus-collector containers on cluster ci-dev-aks-mac-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-mac-eus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-mac-eus\"}) )) by (container, pod) > 0.5", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "CPU usage greater than 5% for prometheus-collector on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-mac-eus", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-mac-eus\", namespace=\"monitoring\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{cluster=\"ci-dev-aks-mac-eus\", namespace=\"monitoring\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-dev-aks-mac-eus", - "expression": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"})> 0.00025", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-dev-aks-mac-eus", - "expression": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"}) > 0.0008", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for k-s-m on cluster ci-dev-aks-mac-eus", - "expression": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"}) > 0.00001183", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for kube state metrics on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for k-s-m on cluster ci-dev-aks-mac-eus", - "expression": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"}) > 0.00023", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for kube state metrics on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-dev-aks-mac-eus", - "expression": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"}) > 0.00225", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-dev-aks-mac-eus", - "expression": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"}) > 0.0062", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - } - ] - } - } - ] -} \ No newline at end of file diff --git a/internal/alerts/ci-dev-aks-wcus.json b/internal/alerts/ci-dev-aks-wcus.json deleted file mode 100644 index 3d439ac85..000000000 --- a/internal/alerts/ci-dev-aks-wcus.json +++ /dev/null @@ -1,459 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "name": "containerinsights_dev_wcus_alerts", - "type": "Microsoft.AlertsManagement/prometheusRuleGroups", - "apiVersion": "2023-03-01", - "location": "westcentralus", - "properties": { - "description": "rule group for cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-wcus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-wcus in MAC: /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-wcus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-wcus-mac", - "scopes": [ - "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-wcus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-wcus-mac" - ], - "rules": [ - { - "alert": "Amd64 metric missing in cluster ci-dev-aks-wcus", - "expression": "absent(node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"x86_64\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Amd64 metric missing in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = node in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"node\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = node in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kubelet in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kubelet\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kubelet in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = windows-exporter in cluster ci-dev-aks-wcus", - "expression": "up{cluster=\"ci-dev-aks-wcus\", job=\"windows-exporter\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = windows-exporter in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-apiserver in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-apiserver\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-apiserver in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy-windows in cluster ci-dev-aks-wcus", - "expression": "up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy-windows\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-state-metrics in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-state-metrics\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-state-metrics in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = cadvisor in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"cadvisor\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = cadvisor in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-dns in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-dns\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-dns in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-wcus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-wcus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-wcus\"}) )) by (container, pod) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "CPU usage greater than 90% for prometheus-collector on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 5 for prometheus-collector containers on cluster ci-dev-aks-wcus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-wcus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-wcus\"}) )) by (container, pod) > 0.5", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "CPU usage greater than 5% for prometheus-collector on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-wcus", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-dev-aks-wcus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) - sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.103", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-dev-aks-wcus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.0078", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for k-s-m on cluster ci-dev-aks-wcus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{ image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*kube-state-metrics.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.00014", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for kube state metrics on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for k-s-m on cluster ci-dev-aks-wcus", - "expression": "sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*kube-state-metrics.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.002", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for kube state metrics on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-dev-aks-wcus", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector.*\"}) - sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"})) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.0157", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-dev-aks-wcus", - "expression": "sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.033", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - } - ] - } - } - ] -} \ No newline at end of file diff --git a/internal/alerts/ci-prod-aks-mac-weu.json b/internal/alerts/ci-prod-aks-mac-weu.json deleted file mode 100644 index 4d8a119eb..000000000 --- a/internal/alerts/ci-prod-aks-mac-weu.json +++ /dev/null @@ -1,436 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "name": "containerinsights_prod_mac_weu_alerts", - "type": "Microsoft.AlertsManagement/prometheusRuleGroups", - "apiVersion": "2023-03-01", - "location": "westeurope", - "properties": { - "description": "rule group for cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu in MAC: /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.Monitor/accounts/ci-prod-aks-weu-mac", - "scopes": [ - "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.Monitor/accounts/ci-prod-aks-weu-mac" - ], - "rules": [ - { - "alert": "Amd64 metric missing in cluster ci-prod-aks-mac-weu", - "expression": "absent(node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "Amd64 metric missing in cluster ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = node in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = node in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kubelet in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kubelet\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kubelet in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = windows-exporter in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"windows-exporter\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"windows-exporter\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = windows-exporter in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-apiserver in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-apiserver\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-apiserver in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy-windows in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy-windows\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy-windows\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-state-metrics in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = cadvisor in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = cadvisor in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-dns in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-dns\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-dns in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu\"}) )) by (container, pod) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "CPU usage greater than 90% for prometheus-collector on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Memory usage % greater than 90 for prometheus-collector containers on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu\", namespace=\"monitoring\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{cluster=\"/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu\", namespace=\"monitoring\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "Memory usage % greater than 90 for prometheus-collector containers on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for ama-metrics replicaset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"}) > 0.000200", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "CPU usage % exceeded for ama-metrics replicaset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for ama-metrics daemonset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"}) > 0.000510", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "CPU usage % exceeded for ama-metrics daemonset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for ama-metrics kube state metrics on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"}) > 0.00000953", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": " CPU usage % exceeded for ama-metrics kube state metrics on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for ama-metrics kube state metrics on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"}) > 0.000230", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": " Memory usage % exceeded for ama-metrics kube state metrics on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for ama-metrics daemonset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"}) > 0.00740", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "Memory usage % exceeded for ama-metrics daemonset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for ama-metrics replicaset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"}) > 0.00265", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "Memory usage % exceeded for ama-metrics replicaset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - } - ] - } - } - ] -} \ No newline at end of file diff --git a/internal/alerts/ci-prod-aks-msi-eus2.json b/internal/alerts/ci-prod-aks-msi-eus2.json deleted file mode 100644 index c545bc91c..000000000 --- a/internal/alerts/ci-prod-aks-msi-eus2.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "name": "containerinsightsprodclusteraccount_alerts_2", - "type": "Microsoft.AlertsManagement/prometheusRuleGroups", - "apiVersion": "2023-03-01", - "location": "eastus2euap", - "properties": { - "description": "rule group for cluster ci-prod-aks-msi-eus2 in containerinsightsprodclusteraccount", - "scopes": [ - "/subscriptions/ad2f1a83-caac-4e21-9d2a-9ca3f87105e2/resourcegroups/divyaj-test/providers/microsoft.monitor/accounts/containerinsightsprodclusteraccounteus2euap" - ], - "rules": [ - { - "alert": "up metric missing for target = node in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"node\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = node in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kubelet in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kubelet\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kubelet in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = windows-exporter in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"windows-exporter\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = windows-exporter in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kube-proxy\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kube-apiserver\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-apiserver in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy-windows in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kube-proxy-windows\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-state-metrics in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kube-state-metrics\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = cadvisor in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"cadvisor\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = cadvisor in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-dns in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kube-dns\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-dns in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-msi-eus2", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-msi-eus2\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-msi-eus2\"}) )) by (container, pod) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "CPU usage greater than 90% for prometheus-collector on cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-msi-eus2", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-msi-eus2\", namespace=\"monitoring\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{cluster=\"ci-prod-aks-msi-eus2\", namespace=\"monitoring\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - } - ] - } - } - ] -} \ No newline at end of file diff --git a/internal/alerts/ci_prod_aks_eus.json b/internal/alerts/ci_prod_aks_eus.json deleted file mode 100644 index e6ffb3056..000000000 --- a/internal/alerts/ci_prod_aks_eus.json +++ /dev/null @@ -1,459 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "name": "containerinsights_prod_eus_alerts", - "type": "Microsoft.AlertsManagement/prometheusRuleGroups", - "apiVersion": "2023-03-01", - "location": "eastus", - "properties": { - "description": "rule group for cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-prod-aks-eus in MAC: /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-eus-rg/providers/microsoft.monitor/accounts/ci-prod-aks-eus-mac", - "scopes": [ - "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-eus-rg/providers/microsoft.monitor/accounts/ci-prod-aks-eus-mac" - ], - "rules": [ - { - "alert": "Amd64 metric missing in cluster ci-prod-aks-eus", - "expression": "absent(node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"x86_64\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Amd64 metric missing in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = node in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"node\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = node in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kubelet in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kubelet\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kubelet in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = windows-exporter in cluster ci-prod-aks-eus", - "expression": "up{cluster=\"ci-prod-aks-eus\", job=\"windows-exporter\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = windows-exporter in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-apiserver\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-apiserver in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy-windows in cluster ci-prod-aks-eus", - "expression": "up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy-windows\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-state-metrics in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-state-metrics\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-state-metrics in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = cadvisor in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"cadvisor\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = cadvisor in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-dns in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-dns\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-dns in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-eus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-eus\"}) )) by (container, pod) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "CPU usage greater than 90% for prometheus-collector on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 5 for prometheus-collector containers on cluster ci-prod-aks-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-eus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-eus\"}) )) by (container, pod) > 0.5", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "CPU usage greater than 5% for prometheus-collector on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-eus", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-prod-aks-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) - sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.28", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-prod-aks-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.0105", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for k-s-m on cluster ci-prod-aks-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{ image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*kube-state-metrics.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.000238", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for kube state metrics on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for k-s-m on cluster ci-prod-aks-eus", - "expression": "sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*kube-state-metrics.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.0022", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for kube state metrics on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-prod-aks-eus", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector.*\"}) - sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"})) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.023", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-prod-aks-eus", - "expression": "sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.0445", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - } - ] - } - } - ] -} \ No newline at end of file diff --git a/internal/alerts/cluster-scopes.txt b/internal/alerts/cluster-scopes.txt new file mode 100644 index 000000000..d9b5629ae --- /dev/null +++ b/internal/alerts/cluster-scopes.txt @@ -0,0 +1,24 @@ +Here are the cluster parameters which need to be updated in example-alert-template.json before deploying the alerts templates for each cluster. Please +update the "scopes" field in the alerts template with the cluster id and AMW id from the list below depending on the cluster. +Update the clusterName field with the cluster name below. Update the location according to the cluster. Update the alert name accordingly. + +Cluster name Cluster id + + +ci-dev-aks-mac-eus /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-mac-eus +ci-dev-arc-wcus /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-arc-wcus/providers/Microsoft.ContainerService/managedClusters/ci-dev-arc-wcus +ci-prod-aks-mac-weu /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.ContainerService/managedClusters/ci-prod-aks-mac-weu +ci-prod-arc-wcus /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.ContainerService/managedClusters/ci-prod-arc-wcus +monitoring-metrics-prod-aks-eus2euap /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/Microsoft.ContainerService/managedClusters/monitoring-metrics-prod-aks-eus2euap +monitoring-metrics-prod-aks-wcus /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/Microsoft.ContainerService/managedClusters/monitoring-metrics-prod-aks-wcus + + +Azure Monitor Workspace Location +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-eus-mac eastus +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-arc-wcus/providers/microsoft.monitor/accounts/ci-dev-arc-amw westcentralus +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.Monitor/accounts/ci-prod-aks-weu-mac westeurope +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/microsoft.monitor/accounts/ci-prod-arc-wcus westcentralus +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-amw/providers/microsoft.monitor/accounts/monitoring-metrics-amw-eus2euap eastus2euap +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-amw/providers/microsoft.monitor/accounts/monitoring-metrics-amw-wcus westcentralus + + diff --git a/internal/alerts/example-alert-template.json b/internal/alerts/example-alert-template.json new file mode 100644 index 000000000..ce865c59d --- /dev/null +++ b/internal/alerts/example-alert-template.json @@ -0,0 +1,259 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "containerinsights_dev_mac_eus_alerts", + "type": "Microsoft.AlertsManagement/prometheusRuleGroups", + "apiVersion": "2023-03-01", + "location": "eastus", + "properties": { + "description": "rule group for cluster ci-dev-aks-mac-eus in MAC: ci-dev-aks-eus-mac", + "scopes": [ + "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-eus-mac", + "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-mac-eus" + ], + "enabled": true, + "clusterName": "ci-dev-aks-mac-eus", + "rules": [ + { + "alert": "Amd64 metric missing in cluster ci-dev-aks-mac-eus", + "expression": "absent(node_uname_info{machine=\"x86_64\"}) == 1 or node_uname_info{machine=\"x86_64\"} == 0", + "for": "PT3M", + "annotations": { + "description": "Amd64 metric missing in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = node in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"node\"}) == 1 or up{job=\"node\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = node in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kubelet in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kubelet\"}) == 1 or up{job=\"kubelet\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kubelet in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = windows-exporter in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"windows-exporter\"}) == 1 or up{job=\"windows-exporter\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = windows-exporter in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kube-proxy in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kube-proxy\"}) == 1 or up{job=\"kube-proxy\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kube-proxy in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kube-apiserver in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kube-apiserver\"}) == 1 or up{job=\"kube-apiserver\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kube-apiserver in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kube-proxy-windows\"}) == 1 or up{job=\"kube-proxy-windows\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kube-state-metrics\"}) == 1 or up{job=\"kube-state-metrics\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = cadvisor in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"cadvisor\"}) == 1 or up{job=\"cadvisor\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = cadvisor in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kube-dns in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kube-dns\"}) == 1 or up{job=\"kube-dns\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kube-dns in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-mac-eus", + "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"kube-system\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"kube-system\"}) )) by (container, pod) > 0.9", + "for": "PT3M", + "annotations": { + "description": "CPU usage greater than 90% for prometheus-collector on cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "CPU usage % greater than 50 for prometheus-collector containers on cluster ci-dev-aks-mac-eus", + "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"kube-system\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"kube-system\"}) )) by (container, pod) > 0.5", + "for": "PT3M", + "annotations": { + "description": "CPU usage greater than 5% for prometheus-collector on cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "Memory usage is high for prometheus-collector containers on cluster ci-dev-aks-mac-eus", + "expression": "(sum(container_memory_working_set_bytes{namespace=\"kube-system\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{namespace=\"kube-system\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 1.9", + "for": "PT3M", + "annotations": { + "description": "Memory usage is high for prometheus-collector containers on cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + } + ] + } + } + ] +} diff --git a/internal/monitoring/README.md b/internal/monitoring/README.md index 87c9d0f0e..7c50ef28e 100644 --- a/internal/monitoring/README.md +++ b/internal/monitoring/README.md @@ -1,32 +1,29 @@ -### **This wiki contains links of all the resources related to alerts and dashboards of the CI CD clusters** +### **This wiki contains links of all the resources related to alerts and dashboards of the CI CD and prod monitoring near ring clusters** -**Links to all AKS clusters used in CI CD** +Below is the linking of the AKS cluster to Azure Monitor Workspace to Grafana for cicd and prod monitoring clusters: -Below are the links to all the AKS clusters used in CI CD. -We have 2 dev clusters and 2 prod clusters. +ci/cd clusters (cluster --> amw --> grafana) +============================================ -* ci-prod-aks-eus - [link](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-prod-aks-eus/overview) +[AKS] +dev=[ci-dev-aks-mac-eus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-mac-eus/overview) --> [ci-dev-aks-eus-mac](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-eus-mac/resourceOverviewId) --> [cicd-graf-metrics-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.Dashboard/grafana/cicd-graf-metrics-wcus/overview) -* ci-prod-aks-mac-weu - [link](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.ContainerService/managedClusters/ci-prod-aks-mac-weu/overview) +prod=[ci-prod-aks-mac-weu](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.ContainerService/managedClusters/ci-prod-aks-mac-weu/overview) --> [ci-prod-aks-weu-mac](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/microsoft.monitor/accounts/ci-prod-aks-weu-mac/resourceOverviewId) --> [cicd-graf-metrics-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.Dashboard/grafana/cicd-graf-metrics-wcus/overview) -* ci-dev-aks-mac-eus - [link](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-mac-eus/overview) +[ARC] +dev=[ci-dev-arc-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-arc-wcus/providers/Microsoft.ContainerService/managedClusters/ci-dev-arc-wcus/overview) --> [ci-dev-arc-amw](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-arc-wcus/providers/microsoft.monitor/accounts/ci-dev-arc-amw/resourceOverviewId) --> [cicd-graf-metrics-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.Dashboard/grafana/cicd-graf-metrics-wcus/overview) -* ci-dev-aks-wcus - [link](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-wcus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-wcus/overview) +prod=[ci-prod-arc-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.ContainerService/managedClusters/ci-prod-arc-wcus/overview)--> [ci-prod-arc-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/microsoft.monitor/accounts/ci-prod-arc-wcus/resourceOverviewId) --> [cicd-graf-metrics-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.Dashboard/grafana/cicd-graf-metrics-wcus/overview) -**Links of Grafana Instances** -* ci-prod-aks-eus - [link](https://ci-prod-aks-eus-graf-dueya5aadyd2ghef.eus.grafana.azure.com) +canary/prod monitoring clusters (cluster --> amw -->grafana) +=========================================================== -* ci-prod-aks-mac-weu - [link](https://ci-prod-aks-weu-graf-fffvdrhqgkg6dxgm.weu.grafana.azure.com) +[monitoring-metrics-prod-aks-eus2euap](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/Microsoft.ContainerService/managedClusters/monitoring-metrics-prod-aks-eus2euap/overview) --> [monitoring-metrics-amw-eus2euap](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-amw/providers/microsoft.monitor/accounts/monitoring-metrics-amw-eus2euap/resourceOverviewId) --> [monitoring-grafana-metrics-westus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/microsoft.dashboard/grafana/mon-graf-metric-westus/overview) +[monitoring-metrics-prod-aks-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/Microsoft.ContainerService/managedClusters/monitoring-metrics-prod-aks-wcus/overview) --> [monitoring-metrics-amw-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-amw/providers/microsoft.monitor/accounts/monitoring-metrics-amw-wcus/resourceOverviewId) --> [monitoring-grafana-metrics-westus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/microsoft.dashboard/grafana/mon-graf-metric-westus/overview) -* ci-dev-aks-mac-eus - [link](https://ci-dev-aks-eus-graf-aje3bpf7d3ctc9h6.eus.grafana.azure.com) -* ci-dev-aks-wcus - [link](https://ci-dev-aks-wcus-graf-acfcb0a6emefghgn.wcus.grafana.azure.com) +**Dashboard for CI CD and prod monitoring clusters** -**Dashboard fo CI CD alerts** -* ci-prod-aks-eus - [link](https://ci-prod-aks-eus-graf-dueya5aadyd2ghef.eus.grafana.azure.com/d/dmzAODN4k/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1) +* CICD - [link](https://cicd-graf-metrics-wcus-dkechtfecuadeuaw.wcus.grafana.azure.com/d/gp9556IVy/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1) -* ci-prod-aks-mac-weu - [link](https://ci-prod-aks-weu-graf-fffvdrhqgkg6dxgm.weu.grafana.azure.com/d/gp9556IVz/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1) - -* ci-dev-aks-mac-eus - [link](https://ci-dev-aks-eus-graf-aje3bpf7d3ctc9h6.eus.grafana.azure.com/d/gp9556IVz/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1&from=1667414623523&to=1667457823523) - -* ci-dev-aks-wcus - [link](https://ci-dev-aks-wcus-graf-acfcb0a6emefghgn.wcus.grafana.azure.com/d/dmzAODN4k/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1) +* Prod near ring - [link](https://mon-graf-metric-westus-f5hvdcaxc3hjdcdm.wus.grafana.azure.com/d/gp9556IVy/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1) diff --git a/internal/monitoring/dashboards/ci-dev-aks-mac-eus-db.json b/internal/monitoring/dashboards/ci-dev-aks-mac-eus-db.json deleted file mode 100644 index fe3bd4c72..000000000 --- a/internal/monitoring/dashboards/ci-dev-aks-mac-eus-db.json +++ /dev/null @@ -1,1631 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 58, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 34, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"} == 0\r\n\r\n", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Amd64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "{__name__=\"node_uname_info\", cluster=\"ci-dev-aks-mac-eus\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 32, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"aarch64\"} == 0\r\n\r\n", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Aarch64 metric missing ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00025, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.00025 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"})", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Replicaset CPU %", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.00023 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", - "format": "time_series", - "range": true, - "refId": "A" - } - ], - "title": "K-S-M Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0008, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.0008 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"}) ", - "refId": "A" - } - ], - "title": "Daemonset CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0074, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0062 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Daemonset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00001183, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.00000953 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"})", - "refId": "A" - } - ], - "title": "K-S-M CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00265, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.00225 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Replicaset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = cadvisor in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-dns\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-dns in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy-windows\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Up{cluster=\"ci-dev-aks-mac-eus\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-apiserver\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kubelet \"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kubelet in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 57 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"windows-exporter\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = windows-exporter in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 65 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = node in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-12h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", - "uid": "gp9556IVz", - "version": 15, - "weekStart": "" -} \ No newline at end of file diff --git a/internal/monitoring/dashboards/ci-dev-aks-wcus-db.json b/internal/monitoring/dashboards/ci-dev-aks-wcus-db.json deleted file mode 100644 index 8852bc26d..000000000 --- a/internal/monitoring/dashboards/ci-dev-aks-wcus-db.json +++ /dev/null @@ -1,1634 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 58, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "{__name__=\"node_uname_info\", cluster=\"ci-dev-aks-wcus\", domainname=\"(none)\", instance=\"aks-lrnm-38217791-vmss000004\", job=\"node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-lrnm-38217791-vmss000004\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 32, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"x86_64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Amd64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 34, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"aarch64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "aarch64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00014, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.00014 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{ image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*kube-state-metrics.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"})", - "refId": "A" - } - ], - "title": "K-S-M CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.033, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.033 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Daemonset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0078, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0078 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"})", - "range": true, - "refId": "A" - } - ], - "title": "Daemonset CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.002, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.002 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*kube-state-metrics.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"})", - "range": true, - "refId": "A" - } - ], - "title": "K-S-M Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.103, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.103 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) - sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"})", - "refId": "A" - } - ], - "title": "Replicaset CPU %", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0157, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0157 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "(sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector.*\"}) - sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"})) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Replicaset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Up{cluster=\"ci-dev-aks-wcus\", instance=\"10.240.0.61:9153\", job=\"kube-dns\", pod=\"coredns-59b6bf8b4f-p7c67\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"node\"} == 0\r\n", - "format": "time_series", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = node", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-apiserver\"} == 0\r\n", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-apiserver ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy\"} == 0\r\n", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kubelet\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kubelet", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy-windows\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy-windows\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy-windows", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"windows-exporter\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"windows-exporter\"} == 0\r\n\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = windows-exporter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "\r\nabsent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-state-metrics\"} == 0\r\n", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-state-metrics ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"cadvisor\"} == 0\r\n", - "refId": "A" - } - ], - "title": "up metric missing for target = cadvisor", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 64 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "\r\nabsent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-dns\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-dns ", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", - "uid": "dmzAODN4k", - "version": 10, - "weekStart": "" -} \ No newline at end of file diff --git a/internal/monitoring/dashboards/ci-prod-aks-eus-db.json b/internal/monitoring/dashboards/ci-prod-aks-eus-db.json deleted file mode 100644 index 6adf58d64..000000000 --- a/internal/monitoring/dashboards/ci-prod-aks-eus-db.json +++ /dev/null @@ -1,1625 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 58, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 32, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"x86_64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Amd64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 34, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"aarch64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "aarch64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.000238, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.000238 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{ image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*kube-state-metrics.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"})", - "range": true, - "refId": "A" - } - ], - "title": "K-S-M CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.04, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.04 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"})", - "range": true, - "refId": "A" - } - ], - "title": "Daemonset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0105, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0105 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"})", - "range": true, - "refId": "A" - } - ], - "title": "Daemonset CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0022, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0022 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*kube-state-metrics.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"})", - "range": true, - "refId": "A" - } - ], - "title": "K-S-M Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.28, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.28 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) - sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"})", - "range": true, - "refId": "A" - } - ], - "title": "Replicaset CPU %", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.023, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.023 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "(sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector.*\"}) - sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"})) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"})", - "range": true, - "refId": "A" - } - ], - "title": "Replicaset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Up{cluster=\"cimon-aks-wcus\", instance=\"10.240.0.61:9153\", job=\"kube-dns\", pod=\"coredns-59b6bf8b4f-p7c67\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"node\"} == 0\r\n", - "format": "time_series", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = node", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-apiserver\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-apiserver ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kubelet\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kubelet", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy-windows\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy-windows", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus-mdm" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus-mdm" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-prod-aks-eus\", job=\"windows-exporter\"} == 0 \r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = windows-exporter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "\r\nabsent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-state-metrics\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-state-metrics ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"cadvisor\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = cadvisor", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 64 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "\r\nabsent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-dns\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-dns ", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", - "uid": "dmzAODN4k", - "version": 45, - "weekStart": "" -} \ No newline at end of file diff --git a/internal/monitoring/dashboards/ci-prod-aks-mac-weu-db.json b/internal/monitoring/dashboards/ci-prod-aks-mac-weu-db.json deleted file mode 100644 index f5dd9758b..000000000 --- a/internal/monitoring/dashboards/ci-prod-aks-mac-weu-db.json +++ /dev/null @@ -1,1552 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 58, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 32, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Amd64 metric missing ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 34, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"aarch64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "aarch64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0002, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0002 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"})", - "refId": "A" - } - ], - "title": "Replicaset CPU %", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00023, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.00023 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "K-S-M Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.00071 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"}) ", - "refId": "A" - } - ], - "title": "Daemonset CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0074, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0074 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Daemonset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00000953, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.00000953 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "editorMode": "code", - "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"})", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "K-S-M CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00265, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.00265 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Replicaset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = cadvisor in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-dns\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-dns in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy-windows\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy-windows\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy-windows in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-state-metrics in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-apiserver\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kubelet\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kubelet in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 57 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"windows-exporter\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"windows-exporter\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = windows-exporter in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 65 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = node in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", - "uid": "gp9556IVz", - "version": 25, - "weekStart": "" -} \ No newline at end of file diff --git a/internal/monitoring/dashboards/cicd-db.json b/internal/monitoring/dashboards/cicd-db.json new file mode 100644 index 000000000..547914803 --- /dev/null +++ b/internal/monitoring/dashboards/cicd-db.json @@ -0,0 +1,8838 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 42, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 84, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 0.00095, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"ci-prod-arc-wcus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-arc-wcus\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"ci-prod-arc-wcus\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "absent(up{cluster=\"ci-prod-arc-wcus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-arc-wcus\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "absent(up{cluster=\"ci-prod-arc-wcus\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-arc-wcus\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"ci-prod-arc-wcus\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-prod-arc-wcus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-prod-arc-wcus\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-prod-arc-wcus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-arc-wcus\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 90 + }, + "id": 132, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "cluster = ci-prod-arc-wcus", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 35, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 109, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 110, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 111, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 112, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 113, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 114, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 115, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 116, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 118, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 120, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 121, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 122, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"ci-dev-arc-wcus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-arc-wcus\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"ci-dev-arc-wcus\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 123, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "absent(up{cluster=\"ci-dev-arc-wcus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-arc-wcus\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 124, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 125, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 126, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 127, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 128, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 129, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "absent(up{cluster=\"ci-dev-arc-wcus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-arc-wcus\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"ci-dev-arc-wcus\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 130, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-dev-arc-wcus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-dev-arc-wcus\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 131, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-dev-arc-wcus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-arc-wcus\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 133, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "Cluster = ci-dev-arc-wcus", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 60, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 0.00095, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 86, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 90, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 91, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 92, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 93, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 94, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 95, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 96, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 97, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 98, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 99, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"ci-prod-aks-mac-weu\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 60 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 60 + }, + "id": 101, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 102, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 68 + }, + "id": 103, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 104, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 105, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 84 + }, + "id": 106, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"ci-prod-aks-mac-weu\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 84 + }, + "id": 107, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 92 + }, + "id": 108, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 92 + }, + "id": 134, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "Cluster = ci-prod-aks-mac-weu", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 85, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 0.00095, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"ci-dev-aks-mac-eus\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 61 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 61 + }, + "id": 76, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 69 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 81, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"ci-dev-aks-mac-eus\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 82, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 93 + }, + "id": 83, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 93 + }, + "id": 135, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "Cluster = ci-dev-aks-mac-eus", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", + "uid": "gp9556IVy", + "version": 38, + "weekStart": "" + } diff --git a/internal/monitoring/dashboards/prod-near-ring-db.json b/internal/monitoring/dashboards/prod-near-ring-db.json new file mode 100644 index 000000000..a96f5f673 --- /dev/null +++ b/internal/monitoring/dashboards/prod-near-ring-db.json @@ -0,0 +1,4444 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 44, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 35, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 109, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 110, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 111, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 112, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 113, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 114, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 115, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 116, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 118, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 120, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 121, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 122, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"cadvisor\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"monitoring-metrics-prod-aks-wcus\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 123, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 124, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 125, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 126, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 127, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 128, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 129, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"node\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"monitoring-metrics-prod-aks-wcus\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 130, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"monitoring-metrics-prod-aks-wcus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"monitoring-metrics-prod-aks-wcus\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 131, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"monitoring-metrics-prod-aks-wcus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"monitoring-metrics-prod-aks-wcus\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 90 + }, + "id": 132, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "Cluster = monitoring-metrics-prod-aks-wcus", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 84, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 0.00095, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"cadvisor\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"node\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"monitoring-metrics-prod-aks-eus2euap\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"monitoring-metrics-prod-aks-eus2euap\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"monitoring-metrics-prod-aks-eus2euap\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"monitoring-metrics-prod-aks-eus2euap\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 133, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "cluster = monitoring-metrics-amw-eus2euap", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", + "uid": "gp9556IVy", + "version": 13, + "weekStart": "" + }