From 8c55db7b28047a037c18102aabdc99c7a5b67138 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:58:55 +0000 Subject: [PATCH 1/5] Bump github.com/prometheus/client_golang Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.16.0 to 1.17.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/v1.17.0/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.16.0...v1.17.0) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- otelcollector/fluent-bit/src/go.mod | 2 +- otelcollector/fluent-bit/src/go.sum | 30 +++++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/otelcollector/fluent-bit/src/go.mod b/otelcollector/fluent-bit/src/go.mod index a6f6f31f4..a741819ca 100644 --- a/otelcollector/fluent-bit/src/go.mod +++ b/otelcollector/fluent-bit/src/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/fluent/fluent-bit-go v0.0.0-20220311094233-780004bf5562 github.com/microsoft/ApplicationInsights-Go v0.4.4 - github.com/prometheus/client_golang v1.16.0 + github.com/prometheus/client_golang v1.17.0 github.com/rogpeppe/go-internal v1.11.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 diff --git a/otelcollector/fluent-bit/src/go.sum b/otelcollector/fluent-bit/src/go.sum index 44a3d7222..f980adc7b 100644 --- a/otelcollector/fluent-bit/src/go.sum +++ b/otelcollector/fluent-bit/src/go.sum @@ -38,6 +38,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE= +github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -285,29 +286,34 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= @@ -340,6 +346,7 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -472,8 +479,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -531,8 +538,10 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -702,8 +711,9 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e9c13baa86ed0ccae0a5f38477e8dd2c0c4104f0 Mon Sep 17 00:00:00 2001 From: Vishwanath Date: Thu, 5 Oct 2023 21:57:20 -0700 Subject: [PATCH 2/5] merge from main (#622) * Add cluster scope to alert rule groups for linking them with UX (#600) * Removing duplicate alerts from ci recommended alerts * Remove test branch * Remove preview keyword from policy readme * Add cluster id scope to rule groups for alerts to be linked to UX * adding terraform update * Add cicd and prod near ring cluster monitoring for managed prometheus (#602) * Removing duplicate alerts from ci recommended alerts * Remove test branch * Remove preview keyword from policy readme * Add cicd and prod near ring cluster monitoring for managed prometheus * Make single file for alerts and update dashboard to include unit in memory * Replace label for cluster name with clusterName property * Add telemetry for scrape interval (#614) * Removing duplicate alerts from ci recommended alerts * Remove test branch * Remove preview keyword from policy readme * Add telemetry for scrape interval hash * remove branch name * Upgrade dependencies (#616) * upgrade all dependencies but collector * update golang * Update CVEs * update config * remove branch * make build fail when trivy fails * fix trivy scan for image not found for PRs * windows fix for replicaset collecting windows data (#620) * fix: remove uneeded windows scrape config in replicaset * . * . * . * revert windows telegraf update * missed end * remove examplar disablement for windows * bin place me_win configs --------- Co-authored-by: Grace Wehner Co-authored-by: bragi92 * Bug fix- update cert thumbprint to latest ame prod cert (#615) * Removing duplicate alerts from ci recommended alerts * Remove test branch * Remove preview keyword from policy readme * Bug fix- update cert thumbprint for image signing to latest ame prod cert * Fix $ substitution issue in relabel and metric relabel config (#618) * dollar fix for node name and node ip * test $ replacement * clean up build * Bump @adobe/css-tools in /tools/az-prom-rules-converter/web-app-example (#577) Bumps [@adobe/css-tools](https://github.com/adobe/css-tools) from 4.0.1 to 4.3.1. - [Changelog](https://github.com/adobe/css-tools/blob/main/History.md) - [Commits](https://github.com/adobe/css-tools/commits) --- updated-dependencies: - dependency-name: "@adobe/css-tools" dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: Sohamdg081992 <31517098+Sohamdg081992@users.noreply.github.com> Co-authored-by: Grace Wehner Co-authored-by: bragi92 Co-authored-by: rashmichandrashekar Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .pipelines/azure-pipeline-build.yml | 27 +- .trivyignore | 44 +- .../FullAzureMonitorMetricsProfile.json | 12 +- .../WindowsRecordingRules.json | 8 +- .../AzureMonitorAlertsProfile.bicep | 8 +- .../FullAzureMonitorMetricsProfile.bicep | 16 +- AddonBicepTemplate/README.md | 2 +- .../AddonPolicyMetricsProfile.rules.json | 16 +- AddonTerraformTemplate/main.tf | 8 +- .../Default/DefaultAlerts.json | 9 +- .../Default/DefaultRecordingRules.json | 11 +- internal/alerts/ci-dev-aks-mac-eus.json | 459 - internal/alerts/ci-dev-aks-wcus.json | 459 - internal/alerts/ci-prod-aks-mac-weu.json | 436 - internal/alerts/ci-prod-aks-msi-eus2.json | 275 - internal/alerts/ci_prod_aks_eus.json | 459 - internal/alerts/cluster-scopes.txt | 24 + internal/alerts/example-alert-template.json | 259 + internal/docs/ESRPCodeSign.md | 2 +- internal/monitoring/README.md | 37 +- .../dashboards/ci-dev-aks-mac-eus-db.json | 1631 --- .../dashboards/ci-dev-aks-wcus-db.json | 1634 --- .../dashboards/ci-prod-aks-eus-db.json | 1625 --- .../dashboards/ci-prod-aks-mac-weu-db.json | 1552 --- internal/monitoring/dashboards/cicd-db.json | 8838 +++++++++++++++++ .../dashboards/prod-near-ring-db.json | 4444 +++++++++ .../templates/ci_recommended_alerts.json | 9 +- otelcollector/build/windows/Dockerfile | 2 +- otelcollector/build/windows/scripts/main.ps1 | 8 +- .../prometheus-config-merger.rb | 16 - .../templates/ama-metrics-daemonset.yaml | 2 +- .../templates/ama-metrics-deployment.yaml | 2 +- .../values-template.yaml | 2 +- otelcollector/fluent-bit/src/telemetry.go | 177 +- otelcollector/metricextension/me_ds.config | 1 + .../metricextension/me_ds_internal.config | 1 + .../metricextension/me_ds_internal_win.config | 26 + .../metricextension/me_ds_win.config | 26 + .../metricextension/me_internal.config | 1 + .../prom-config-validator-builder/main.go | 9 + otelcollector/scripts/setup.sh | 6 +- .../web-app-example/package-lock.json | 9 +- 42 files changed, 13901 insertions(+), 8691 deletions(-) delete mode 100644 internal/alerts/ci-dev-aks-mac-eus.json delete mode 100644 internal/alerts/ci-dev-aks-wcus.json delete mode 100644 internal/alerts/ci-prod-aks-mac-weu.json delete mode 100644 internal/alerts/ci-prod-aks-msi-eus2.json delete mode 100644 internal/alerts/ci_prod_aks_eus.json create mode 100644 internal/alerts/cluster-scopes.txt create mode 100644 internal/alerts/example-alert-template.json delete mode 100644 internal/monitoring/dashboards/ci-dev-aks-mac-eus-db.json delete mode 100644 internal/monitoring/dashboards/ci-dev-aks-wcus-db.json delete mode 100644 internal/monitoring/dashboards/ci-prod-aks-eus-db.json delete mode 100644 internal/monitoring/dashboards/ci-prod-aks-mac-weu-db.json create mode 100644 internal/monitoring/dashboards/cicd-db.json create mode 100644 internal/monitoring/dashboards/prod-near-ring-db.json create mode 100644 otelcollector/metricextension/me_ds_internal_win.config create mode 100644 otelcollector/metricextension/me_ds_win.config diff --git a/.pipelines/azure-pipeline-build.yml b/.pipelines/azure-pipeline-build.yml index ebee81b37..4bf1da9d4 100644 --- a/.pipelines/azure-pipeline-build.yml +++ b/.pipelines/azure-pipeline-build.yml @@ -2,6 +2,7 @@ trigger: branches: include: - main + - vishwa/10022023-upgrade pr: autoCancel: true branches: @@ -235,7 +236,7 @@ jobs: - task: GoTool@0 displayName: "Build: specify golang version" inputs: - version: '1.19' + version: '1.20' - bash: | sudo apt-get install build-essential -y @@ -336,7 +337,7 @@ jobs: oras attach $(LINUX_FULL_IMAGE_NAME) \ --artifact-type 'application/vnd.cncf.notary.signature' \ ./payload.json:application/cose \ - -a "io.cncf.notary.x509chain.thumbprint#S256=[\"659AAA9C0E822B4B20A964AA0178BD9419A50530\"]" + -a "io.cncf.notary.x509chain.thumbprint#S256=[\"79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\"]" workingDirectory: $(Build.ArtifactStagingDirectory)/linux/ displayName: "ORAS Push Artifacts in $(Build.ArtifactStagingDirectory)/linux/" condition: eq(variables.IS_MAIN_BRANCH, true) @@ -344,10 +345,20 @@ jobs: - bash: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin trivy image --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM --exit-code 1 $(LINUX_FULL_IMAGE_NAME) + if [ $? -ne 0 ]; then + exit 1 + fi trivy image --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM --exit-code 1 $(KUBE_STATE_METRICS_IMAGE) + if [ $? -ne 0 ]; then + exit 1 + fi trivy image --ignore-unfixed --no-progress --severity HIGH,CRITICAL,MEDIUM --exit-code 1 $(NODE_EXPORTER_IMAGE) + if [ $? -ne 0 ]; then + exit 1 + fi workingDirectory: $(Build.SourcesDirectory) displayName: "Build: run trivy scan" + condition: eq(variables.IS_PR, false) - task: CodeQL3000Finalize@0 displayName: 'SDL: run codeql' @@ -423,7 +434,7 @@ jobs: - task: GoTool@0 displayName: "Build: specify golang version" inputs: - version: '1.19' + version: '1.20' - powershell: | ./makefile_windows.ps1 @@ -461,7 +472,7 @@ jobs: - task: GoTool@0 displayName: "Build: specify golang version" inputs: - version: '1.19' + version: '1.20' - powershell: | ./makefile_windows.ps1 @@ -504,7 +515,7 @@ jobs: - task: GoTool@0 displayName: "Build: specify golang version" inputs: - version: '1.19' + version: '1.20' - powershell: | New-Item -Path "$(Build.ArtifactStagingDirectory)" -Name "windows" -ItemType "directory" @@ -570,7 +581,7 @@ jobs: New-Item -ItemType Directory -Force -Path $env:USERPROFILE\bin Copy-Item -Path $currentDirectory\oras.exe -Destination "$env:USERPROFILE\bin\" $env:PATH = "$env:USERPROFILE\bin;$env:PATH" - oras attach $(WINDOWS_FULL_IMAGE_NAME) --artifact-type application/vnd.cncf.notary.signature ./payload.json:application/cose -a io.cncf.notary.x509chain.thumbprint#S256=[\""659AAA9C0E822B4B20A964AA0178BD9419A50530\""] + oras attach $(WINDOWS_FULL_IMAGE_NAME) --artifact-type application/vnd.cncf.notary.signature ./payload.json:application/cose -a io.cncf.notary.x509chain.thumbprint#S256=[\""79E6A702361E1F60DAA84AEEC4CBF6F6420DE6BA\""] workingDirectory: $(Build.ArtifactStagingDirectory)/windows displayName: "Download, install Oras and run oras attach" condition: eq(variables.IS_MAIN_BRANCH, true) @@ -600,7 +611,7 @@ jobs: - task: HelmInstaller@1 displayName: 'Build: install Helm version' inputs: - helmVersionToInstall: latest + helmVersionToInstall: 3.12.3 - bash: | envsubst < $(Build.SourcesDirectory)/otelcollector/deploy/chart/prometheus-collector/Chart-template.yaml > $(Build.SourcesDirectory)/otelcollector/deploy/chart/prometheus-collector/Chart.yaml && envsubst < $(Build.SourcesDirectory)/otelcollector/deploy/chart/prometheus-collector/values-template.yaml > $(Build.SourcesDirectory)/otelcollector/deploy/chart/prometheus-collector/values.yaml @@ -643,7 +654,7 @@ jobs: - task: HelmInstaller@1 displayName: 'Build: install Helm version' inputs: - helmVersionToInstall: latest + helmVersionToInstall: 3.12.3 - bash: | export HELM_CHART_NAME=$ARC_HELM_CHART_NAME diff --git a/.trivyignore b/.trivyignore index be74d628e..4f4245f30 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,29 +1,23 @@ # Check for HIGH/CRITICAL & MEDIUM CVEs. HIGH/CRITICAL to be fixed asap, MEDIUM is best effort # ignore these CVEs, but continue scanning to catch other vulns. Note : this will ignore these cves globally -# CRITICAL/HIGH -# Ruby GEM -CVE-2021-33621 -# node-exporter -CVE-2021-38561 -CVE-2021-44716 -CVE-2022-21698 -CVE-2022-27191 -# opt/telegraf/telegraf -CVE-2022-23471 -CVE-2023-25153 -CVE-2023-25173 +# CRITICAL +# none -# MEDIUM -# opt/telegraf/telegraf -CVE-2019-3826 -# kube-state-metrics -CVE-2022-41723 -# opt/microsoft/otelcollector/otelcollector -# opt/promconfigvalidator -# opt/telegraf/telegraf -# kube-state-metrics -# bin/node_exporter -CVE-2022-41717 -CVE-2022-46146 -CVE-2022-41721 +# =========== HIGH ================ +# HIGH - otelcollector +CVE-2023-2253 +CVE-2023-28840 +# HIGH - promconfigvalidator +CVE-2023-2253 +CVE-2023-28840 + +# =========== MEDIUM ================ +# MEDIUM - otelcollector +CVE-2023-28841 +CVE-2023-28842 +CVE-2023-40577 +# MEDIUM - promconfigvalidator +CVE-2023-28841 +CVE-2023-28842 +CVE-2023-40577 diff --git a/AddonArmTemplate/FullAzureMonitorMetricsProfile.json b/AddonArmTemplate/FullAzureMonitorMetricsProfile.json index 7fb9c5f89..f6432a023 100644 --- a/AddonArmTemplate/FullAzureMonitorMetricsProfile.json +++ b/AddonArmTemplate/FullAzureMonitorMetricsProfile.json @@ -221,7 +221,8 @@ "properties": { "description": "[concat(variables('nodeRecordingRuleGroupDescription'), variables('version'))]", "scopes": [ - "[parameters('azureMonitorWorkspaceResourceId')]" + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" ], "clusterName": "[variables('clusterName')]", "interval": "PT1M", @@ -281,7 +282,8 @@ "properties": { "description": "[concat(variables('kubernetesRecordingRuleGroupDescription'), variables('version'))]", "scopes": [ - "[parameters('azureMonitorWorkspaceResourceId')]" + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" ], "clusterName": "[variables('clusterName')]", "interval": "PT1M", @@ -385,7 +387,8 @@ "properties": { "description": "[concat(variables('RecordingRuleGroupDescriptionWin'), variables('version'))]", "scopes": [ - "[parameters('azureMonitorWorkspaceResourceId')]" + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" ], "enabled": "[parameters('enableWindowsRecordingRules')]", "clusterName": "[variables('clusterName')]", @@ -462,7 +465,8 @@ "properties": { "description": "[concat(variables('RecordingRuleGroupDescriptionWin'), variables('version'))]", "scopes": [ - "[parameters('azureMonitorWorkspaceResourceId')]" + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" ], "enabled": "[parameters('enableWindowsRecordingRules')]", "clusterName": "[variables('clusterName')]", diff --git a/AddonArmTemplate/WindowsRecordingRuleGroupTemplate/WindowsRecordingRules.json b/AddonArmTemplate/WindowsRecordingRuleGroupTemplate/WindowsRecordingRules.json index 68d58498a..69d9e6d49 100644 --- a/AddonArmTemplate/WindowsRecordingRuleGroupTemplate/WindowsRecordingRules.json +++ b/AddonArmTemplate/WindowsRecordingRuleGroupTemplate/WindowsRecordingRules.json @@ -56,7 +56,8 @@ "properties": { "description": "[concat(variables('RecordingRuleGroupDescriptionWin'), variables('version'))]", "scopes": [ - "[parameters('azureMonitorWorkspaceResourceId')]" + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" ], "enabled": true, "clusterName": "[variables('clusterName')]", @@ -133,7 +134,8 @@ "properties": { "description": "[concat(variables('RecordingRuleGroupDescriptionWin'), variables('version'))]", "scopes": [ - "[parameters('azureMonitorWorkspaceResourceId')]" + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" ], "enabled": true, "clusterName": "[variables('clusterName')]", @@ -211,4 +213,4 @@ } } ] -} \ No newline at end of file +} diff --git a/AddonBicepTemplate/AzureMonitorAlertsProfile.bicep b/AddonBicepTemplate/AzureMonitorAlertsProfile.bicep index 7949150c2..6ed9a1b83 100644 --- a/AddonBicepTemplate/AzureMonitorAlertsProfile.bicep +++ b/AddonBicepTemplate/AzureMonitorAlertsProfile.bicep @@ -16,9 +16,7 @@ resource recommendedAlerts 'Microsoft.AlertsManagement/prometheusRuleGroups@2023 location: location properties: { description: 'Kubernetes Alert RuleGroup-RecommendedCIAlerts - 0.1' - scopes: [ - monitorWorkspace.id - ] + scopes: [monitorWorkspace.id,aksResourceId] clusterName: split(aksResourceId, '/')[8] enabled: true interval: 'PT5M' @@ -241,9 +239,7 @@ resource communityALerts 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-0 location: location properties: { description: 'Kubernetes Alert RuleGroup-communityCIAlerts - 0.1' - scopes: [ - monitorWorkspace.id - ] + scopes: [monitorWorkspace.id,aksResourceId] clusterName: split(aksResourceId, '/')[8] enabled: true interval: 'PT1M' diff --git a/AddonBicepTemplate/FullAzureMonitorMetricsProfile.bicep b/AddonBicepTemplate/FullAzureMonitorMetricsProfile.bicep index c906b2e87..58715bc12 100644 --- a/AddonBicepTemplate/FullAzureMonitorMetricsProfile.bicep +++ b/AddonBicepTemplate/FullAzureMonitorMetricsProfile.bicep @@ -141,9 +141,7 @@ resource nodeRecordingRuleGroup 'Microsoft.AlertsManagement/prometheusRuleGroups location: azureMonitorWorkspaceLocation properties: { description: '${nodeRecordingRuleGroupDescription}${version}' - scopes: [ - azureMonitorWorkspaceResourceId - ] + scopes: [azureMonitorWorkspaceResourceId,clusterResourceId] enabled: true clusterName: clusterName interval: 'PT1M' @@ -201,9 +199,7 @@ resource kubernetesRecordingRuleGroup 'Microsoft.AlertsManagement/prometheusRule location: azureMonitorWorkspaceLocation properties: { description: '${kubernetesRecordingRuleGroupDescription}${version}' - scopes: [ - azureMonitorWorkspaceResourceId - ] + scopes: [azureMonitorWorkspaceResourceId,clusterResourceId] enabled: true clusterName: clusterName interval: 'PT1M' @@ -305,9 +301,7 @@ resource nodeRecordingRuleGroupNameWin 'Microsoft.AlertsManagement/prometheusRul location: azureMonitorWorkspaceLocation properties: { description: '${RecordingRuleGroupDescriptionWin}${version}' - scopes: [ - azureMonitorWorkspaceResourceId - ] + scopes: [azureMonitorWorkspaceResourceId,clusterResourceId] enabled: enableWindowsRecordingRules clusterName: clusterName interval: 'PT1M' @@ -381,9 +375,7 @@ resource nodeAndKubernetesRecordingRuleGroupNameWin 'Microsoft.AlertsManagement/ location: azureMonitorWorkspaceLocation properties: { description: '${RecordingRuleGroupDescriptionWin}${version}' - scopes: [ - azureMonitorWorkspaceResourceId - ] + scopes: [azureMonitorWorkspaceResourceId,clusterResourceId] enabled: enableWindowsRecordingRules clusterName: clusterName interval: 'PT1M' diff --git a/AddonBicepTemplate/README.md b/AddonBicepTemplate/README.md index 84c21dccf..85af60539 100644 --- a/AddonBicepTemplate/README.md +++ b/AddonBicepTemplate/README.md @@ -5,7 +5,7 @@ You can deploy the templates using a command like : In order to deploy community alerts and ci recommended alerts through template, deploy using command like: -```az deployment group create -g -n --template-file .\AzureMonitorAlertsProfileParameters.json --parameters .\AzureMonitorAlertsProfileParameters.json``` +```az deployment group create -g -n --template-file .\AzureMonitorAlertsProfile.bicep --parameters .\AzureMonitorAlertsProfileParameters.json``` **NOTE** diff --git a/AddonPolicyTemplate/AddonPolicyMetricsProfile.rules.json b/AddonPolicyTemplate/AddonPolicyMetricsProfile.rules.json index 8137837a0..297d10301 100644 --- a/AddonPolicyTemplate/AddonPolicyMetricsProfile.rules.json +++ b/AddonPolicyTemplate/AddonPolicyMetricsProfile.rules.json @@ -225,7 +225,10 @@ "location": "[parameters('azureMonitorWorkspaceLocation')]", "properties": { "description": "[concat(variables('nodeRecordingRuleGroupDescription'), variables('version'))]", - "scopes": ["[parameters('azureMonitorWorkspaceResourceId')]"], + "scopes": [ + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" + ], "clusterName": "[variables('clusterName')]", "interval": "PT1M", "rules": [ @@ -283,7 +286,10 @@ "location": "[parameters('azureMonitorWorkspaceLocation')]", "properties": { "description": "[concat(variables('kubernetesRecordingRuleGroupDescription'), variables('version'))]", - "scopes": ["[parameters('azureMonitorWorkspaceResourceId')]"], + "scopes": [ + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" + ], "clusterName": "[variables('clusterName')]", "interval": "PT1M", "rules": [ @@ -386,7 +392,8 @@ "properties": { "description": "[concat(variables('RecordingRuleGroupDescriptionWin'), variables('version'))]", "scopes": [ - "[parameters('azureMonitorWorkspaceResourceId')]" + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" ], "enabled": "[parameters('enableWindowsRecordingRules')]", "clusterName": "[variables('clusterName')]", @@ -463,7 +470,8 @@ "properties": { "description": "[concat(variables('RecordingRuleGroupDescriptionWin'), variables('version'))]", "scopes": [ - "[parameters('azureMonitorWorkspaceResourceId')]" + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" ], "enabled": "[parameters('enableWindowsRecordingRules')]", "clusterName": "[variables('clusterName')]", diff --git a/AddonTerraformTemplate/main.tf b/AddonTerraformTemplate/main.tf index 1d386e2d8..b3593d983 100644 --- a/AddonTerraformTemplate/main.tf +++ b/AddonTerraformTemplate/main.tf @@ -119,7 +119,7 @@ resource "azurerm_monitor_alert_prometheus_rule_group" "node_recording_rules_rul description = "Node Recording Rules Rule Group" rule_group_enabled = true interval = "PT1M" - scopes = [azurerm_monitor_workspace.amw.id] + scopes = [azurerm_monitor_workspace.amw.id,azurerm_kubernetes_cluster.k8s.id] rule { enabled = true @@ -209,7 +209,7 @@ resource "azurerm_monitor_alert_prometheus_rule_group" "kubernetes_recording_rul description = "Kubernetes Recording Rules Rule Group" rule_group_enabled = true interval = "PT1M" - scopes = [azurerm_monitor_workspace.amw.id] + scopes = [azurerm_monitor_workspace.amw.id,azurerm_kubernetes_cluster.k8s.id] rule { enabled = true @@ -366,7 +366,7 @@ resource "azurerm_monitor_alert_prometheus_rule_group" "node_and_kubernetes_reco description = "Node and Kubernetes Recording Rules Rule Group for Windows Nodes" rule_group_enabled = true interval = "PT1M" - scopes = [azurerm_monitor_workspace.amw.id] + scopes = [azurerm_monitor_workspace.amw.id,azurerm_kubernetes_cluster.k8s.id] rule { enabled = true @@ -497,7 +497,7 @@ resource "azurerm_monitor_alert_prometheus_rule_group" "node_recording_rules_rul description = "Node and Kubernetes Recording Rules Rule Group for Windows Nodes" rule_group_enabled = true interval = "PT1M" - scopes = [azurerm_monitor_workspace.amw.id] + scopes = [azurerm_monitor_workspace.amw.id,azurerm_kubernetes_cluster.k8s.id] rule { enabled = true diff --git a/GeneratedMonitoringArtifacts/Default/DefaultAlerts.json b/GeneratedMonitoringArtifacts/Default/DefaultAlerts.json index f28eaeb50..d931a73f0 100644 --- a/GeneratedMonitoringArtifacts/Default/DefaultAlerts.json +++ b/GeneratedMonitoringArtifacts/Default/DefaultAlerts.json @@ -8,6 +8,12 @@ "description": "Cluster name" } }, + "clusterResourceId": { + "type": "string", + "metadata": { + "description": "Cluster Resource Id" + } + }, "actionGroupResourceId": { "type": "string", "metadata": { @@ -40,7 +46,8 @@ "properties": { "description": "[concat(variables('kubernetesAlertRuleGroupDescription'), variables('version'))]", "scopes": [ - "[parameters('azureMonitorWorkspaceResourceId')]" + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" ], "clusterName": "[parameters('clusterName')]", "interval": "PT1M", diff --git a/GeneratedMonitoringArtifacts/Default/DefaultRecordingRules.json b/GeneratedMonitoringArtifacts/Default/DefaultRecordingRules.json index 4eb17e815..75054d763 100644 --- a/GeneratedMonitoringArtifacts/Default/DefaultRecordingRules.json +++ b/GeneratedMonitoringArtifacts/Default/DefaultRecordingRules.json @@ -8,6 +8,12 @@ "description": "Cluster name" } }, + "clusterResourceId": { + "type": "string", + "metadata": { + "description": "Cluster Resource Id" + } + }, "azureMonitorWorkspaceResourceId": { "type": "string", "metadata": { @@ -41,7 +47,10 @@ "location": "[parameters('location')]", "properties": { "description": "[concat(variables('nodeRecordingRuleGroupDescription'), variables('version'))]", - "scopes": [ "[parameters('azureMonitorWorkspaceResourceId')]" ], + "scopes": [ + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" + ], "enabled": true, "clusterName": "[parameters('clusterName')]", "interval": "PT1M", diff --git a/internal/alerts/ci-dev-aks-mac-eus.json b/internal/alerts/ci-dev-aks-mac-eus.json deleted file mode 100644 index 3c91fa6a8..000000000 --- a/internal/alerts/ci-dev-aks-mac-eus.json +++ /dev/null @@ -1,459 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "name": "containerinsights_dev_mac_eus_alerts", - "type": "Microsoft.AlertsManagement/prometheusRuleGroups", - "apiVersion": "2023-03-01", - "location": "eastus", - "properties": { - "description": "rule group for cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-mac-eus in MAC: /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-eus-mac", - "scopes": [ - "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-eus-mac" - ], - "rules": [ - { - "alert": "Amd64 metric missing in cluster ci-dev-aks-mac-eus", - "expression": "absent(node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Amd64 metric missing in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = node in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = node in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kubelet in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kubelet\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kubelet in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = windows-exporter in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"windows-exporter\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"windows-exporter\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = windows-exporter in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-apiserver in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-apiserver\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-apiserver in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy-windows\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy-windows\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = cadvisor in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = cadvisor in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-dns in cluster ci-dev-aks-mac-eus", - "expression": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-dns\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-dns in cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-mac-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-mac-eus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-mac-eus\"}) )) by (container, pod) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "CPU usage greater than 90% for prometheus-collector on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 5 for prometheus-collector containers on cluster ci-dev-aks-mac-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-mac-eus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-mac-eus\"}) )) by (container, pod) > 0.5", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "CPU usage greater than 5% for prometheus-collector on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-mac-eus", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-mac-eus\", namespace=\"monitoring\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{cluster=\"ci-dev-aks-mac-eus\", namespace=\"monitoring\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-dev-aks-mac-eus", - "expression": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"})> 0.00025", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-dev-aks-mac-eus", - "expression": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"}) > 0.0008", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for k-s-m on cluster ci-dev-aks-mac-eus", - "expression": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"}) > 0.00001183", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for kube state metrics on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for k-s-m on cluster ci-dev-aks-mac-eus", - "expression": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"}) > 0.00023", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for kube state metrics on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-dev-aks-mac-eus", - "expression": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"}) > 0.00225", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-dev-aks-mac-eus", - "expression": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"}) > 0.0062", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-mac-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-dev-aks-mac-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - } - ] - } - } - ] -} \ No newline at end of file diff --git a/internal/alerts/ci-dev-aks-wcus.json b/internal/alerts/ci-dev-aks-wcus.json deleted file mode 100644 index 3d439ac85..000000000 --- a/internal/alerts/ci-dev-aks-wcus.json +++ /dev/null @@ -1,459 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "name": "containerinsights_dev_wcus_alerts", - "type": "Microsoft.AlertsManagement/prometheusRuleGroups", - "apiVersion": "2023-03-01", - "location": "westcentralus", - "properties": { - "description": "rule group for cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-wcus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-wcus in MAC: /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-wcus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-wcus-mac", - "scopes": [ - "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-wcus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-wcus-mac" - ], - "rules": [ - { - "alert": "Amd64 metric missing in cluster ci-dev-aks-wcus", - "expression": "absent(node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"x86_64\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Amd64 metric missing in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = node in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"node\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = node in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kubelet in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kubelet\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kubelet in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = windows-exporter in cluster ci-dev-aks-wcus", - "expression": "up{cluster=\"ci-dev-aks-wcus\", job=\"windows-exporter\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = windows-exporter in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-apiserver in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-apiserver\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-apiserver in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy-windows in cluster ci-dev-aks-wcus", - "expression": "up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy-windows\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-state-metrics in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-state-metrics\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-state-metrics in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = cadvisor in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"cadvisor\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = cadvisor in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-dns in cluster ci-dev-aks-wcus", - "expression": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-dns\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-dns in cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-wcus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-wcus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-wcus\"}) )) by (container, pod) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "CPU usage greater than 90% for prometheus-collector on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 5 for prometheus-collector containers on cluster ci-dev-aks-wcus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-wcus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-dev-aks-wcus\"}) )) by (container, pod) > 0.5", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "CPU usage greater than 5% for prometheus-collector on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-wcus", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-dev-aks-wcus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) - sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.103", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-dev-aks-wcus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.0078", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for k-s-m on cluster ci-dev-aks-wcus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{ image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*kube-state-metrics.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.00014", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for kube state metrics on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for k-s-m on cluster ci-dev-aks-wcus", - "expression": "sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*kube-state-metrics.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.002", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for kube state metrics on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-dev-aks-wcus", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector.*\"}) - sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"})) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.0157", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-dev-aks-wcus", - "expression": "sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.033", - "for": "PT15M", - "labels": { - "cluster": "ci-dev-aks-wcus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-dev-aks-wcus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - } - ] - } - } - ] -} \ No newline at end of file diff --git a/internal/alerts/ci-prod-aks-mac-weu.json b/internal/alerts/ci-prod-aks-mac-weu.json deleted file mode 100644 index 4d8a119eb..000000000 --- a/internal/alerts/ci-prod-aks-mac-weu.json +++ /dev/null @@ -1,436 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "name": "containerinsights_prod_mac_weu_alerts", - "type": "Microsoft.AlertsManagement/prometheusRuleGroups", - "apiVersion": "2023-03-01", - "location": "westeurope", - "properties": { - "description": "rule group for cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu in MAC: /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.Monitor/accounts/ci-prod-aks-weu-mac", - "scopes": [ - "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.Monitor/accounts/ci-prod-aks-weu-mac" - ], - "rules": [ - { - "alert": "Amd64 metric missing in cluster ci-prod-aks-mac-weu", - "expression": "absent(node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "Amd64 metric missing in cluster ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = node in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = node in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kubelet in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kubelet\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kubelet in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = windows-exporter in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"windows-exporter\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"windows-exporter\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = windows-exporter in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-apiserver in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-apiserver\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-apiserver in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy-windows in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy-windows\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy-windows\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-state-metrics in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = cadvisor in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = cadvisor in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-dns in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-dns\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-dns in cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu\"}) )) by (container, pod) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "CPU usage greater than 90% for prometheus-collector on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Memory usage % greater than 90 for prometheus-collector containers on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu\", namespace=\"monitoring\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{cluster=\"/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu\", namespace=\"monitoring\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "Memory usage % greater than 90 for prometheus-collector containers on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for ama-metrics replicaset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"}) > 0.000200", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "CPU usage % exceeded for ama-metrics replicaset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for ama-metrics daemonset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"}) > 0.000510", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "CPU usage % exceeded for ama-metrics daemonset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for ama-metrics kube state metrics on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"}) > 0.00000953", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": " CPU usage % exceeded for ama-metrics kube state metrics on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for ama-metrics kube state metrics on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"}) > 0.000230", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": " Memory usage % exceeded for ama-metrics kube state metrics on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for ama-metrics daemonset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"}) > 0.00740", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "Memory usage % exceeded for ama-metrics daemonset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for ama-metrics replicaset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu", - "expression": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"}) > 0.00265", - "for": "PT15M", - "labels": { - "cluster": "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "annotations": { - "description": "Memory usage % exceeded for ama-metrics replicaset on cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourcegroups/ci-prod-aks-mac-weu-rg/providers/microsoft.containerservice/managedclusters/ci-prod-aks-mac-weu" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - } - ] - } - } - ] -} \ No newline at end of file diff --git a/internal/alerts/ci-prod-aks-msi-eus2.json b/internal/alerts/ci-prod-aks-msi-eus2.json deleted file mode 100644 index c545bc91c..000000000 --- a/internal/alerts/ci-prod-aks-msi-eus2.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "name": "containerinsightsprodclusteraccount_alerts_2", - "type": "Microsoft.AlertsManagement/prometheusRuleGroups", - "apiVersion": "2023-03-01", - "location": "eastus2euap", - "properties": { - "description": "rule group for cluster ci-prod-aks-msi-eus2 in containerinsightsprodclusteraccount", - "scopes": [ - "/subscriptions/ad2f1a83-caac-4e21-9d2a-9ca3f87105e2/resourcegroups/divyaj-test/providers/microsoft.monitor/accounts/containerinsightsprodclusteraccounteus2euap" - ], - "rules": [ - { - "alert": "up metric missing for target = node in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"node\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = node in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kubelet in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kubelet\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kubelet in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = windows-exporter in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"windows-exporter\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = windows-exporter in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kube-proxy\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kube-apiserver\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-apiserver in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy-windows in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kube-proxy-windows\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-state-metrics in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kube-state-metrics\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = cadvisor in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"cadvisor\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = cadvisor in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-dns in cluster ci-prod-aks-msi-eus2", - "expression": "absent(up{cluster=\"ci-prod-aks-msi-eus2\", job=\"kube-dns\"}) == 1", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-dns in cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-msi-eus2", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-msi-eus2\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-msi-eus2\"}) )) by (container, pod) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "CPU usage greater than 90% for prometheus-collector on cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-msi-eus2", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-msi-eus2\", namespace=\"monitoring\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{cluster=\"ci-prod-aks-msi-eus2\", namespace=\"monitoring\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-msi-eus2" - }, - "annotations": { - "description": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-msi-eus2" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - } - ] - } - } - ] -} \ No newline at end of file diff --git a/internal/alerts/ci_prod_aks_eus.json b/internal/alerts/ci_prod_aks_eus.json deleted file mode 100644 index e6ffb3056..000000000 --- a/internal/alerts/ci_prod_aks_eus.json +++ /dev/null @@ -1,459 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "name": "containerinsights_prod_eus_alerts", - "type": "Microsoft.AlertsManagement/prometheusRuleGroups", - "apiVersion": "2023-03-01", - "location": "eastus", - "properties": { - "description": "rule group for cluster /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-prod-aks-eus in MAC: /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-eus-rg/providers/microsoft.monitor/accounts/ci-prod-aks-eus-mac", - "scopes": [ - "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-eus-rg/providers/microsoft.monitor/accounts/ci-prod-aks-eus-mac" - ], - "rules": [ - { - "alert": "Amd64 metric missing in cluster ci-prod-aks-eus", - "expression": "absent(node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"x86_64\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Amd64 metric missing in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = node in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"node\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = node in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kubelet in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kubelet\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kubelet in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = windows-exporter in cluster ci-prod-aks-eus", - "expression": "up{cluster=\"ci-prod-aks-eus\", job=\"windows-exporter\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = windows-exporter in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-apiserver\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-apiserver in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-proxy-windows in cluster ci-prod-aks-eus", - "expression": "up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy-windows\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-state-metrics in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-state-metrics\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-state-metrics in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = cadvisor in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"cadvisor\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = cadvisor in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "up metric missing for target = kube-dns in cluster ci-prod-aks-eus", - "expression": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-dns\"} == 0", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "up metric is not flowing for target = kube-dns in cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-eus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-eus\"}) )) by (container, pod) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "CPU usage greater than 90% for prometheus-collector on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "CPU usage % greater than 5 for prometheus-collector containers on cluster ci-prod-aks-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-eus\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"monitoring\", cluster=\"ci-prod-aks-eus\"}) )) by (container, pod) > 0.5", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "CPU usage greater than 5% for prometheus-collector on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-eus", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 0.9", - "for": "PT3M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Memory usage % greater than 90 for prometheus-collector containers on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-prod-aks-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) - sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.28", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for replicaset on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-prod-aks-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.0105", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for daemonset on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - CPU usage % exceeded for k-s-m on cluster ci-prod-aks-eus", - "expression": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{ image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*kube-state-metrics.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"}) > 0.000238", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - CPU usage % exceeded for kube state metrics on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for k-s-m on cluster ci-prod-aks-eus", - "expression": "sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*kube-state-metrics.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.0022", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for kube state metrics on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-prod-aks-eus", - "expression": "(sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector.*\"}) - sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"})) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.023", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for replicaset on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - }, - { - "alert": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-prod-aks-eus", - "expression": "sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"}) > 0.0445", - "for": "PT15M", - "labels": { - "cluster": "ci-prod-aks-eus" - }, - "annotations": { - "description": "Build over build alert - Memory usage % exceeded for daemonset on cluster ci-prod-aks-eus" - }, - "severity": 4, - "resolveConfiguration": { - "autoResolved": true, - "timeToResolve": "PT10M" - }, - "actions": [ - { - "ActionProperties": { - "Icm.Enabled": "True" - } - } - ] - } - ] - } - } - ] -} \ No newline at end of file diff --git a/internal/alerts/cluster-scopes.txt b/internal/alerts/cluster-scopes.txt new file mode 100644 index 000000000..d9b5629ae --- /dev/null +++ b/internal/alerts/cluster-scopes.txt @@ -0,0 +1,24 @@ +Here are the cluster parameters which need to be updated in example-alert-template.json before deploying the alerts templates for each cluster. Please +update the "scopes" field in the alerts template with the cluster id and AMW id from the list below depending on the cluster. +Update the clusterName field with the cluster name below. Update the location according to the cluster. Update the alert name accordingly. + +Cluster name Cluster id + + +ci-dev-aks-mac-eus /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-mac-eus +ci-dev-arc-wcus /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-arc-wcus/providers/Microsoft.ContainerService/managedClusters/ci-dev-arc-wcus +ci-prod-aks-mac-weu /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.ContainerService/managedClusters/ci-prod-aks-mac-weu +ci-prod-arc-wcus /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.ContainerService/managedClusters/ci-prod-arc-wcus +monitoring-metrics-prod-aks-eus2euap /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/Microsoft.ContainerService/managedClusters/monitoring-metrics-prod-aks-eus2euap +monitoring-metrics-prod-aks-wcus /subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/Microsoft.ContainerService/managedClusters/monitoring-metrics-prod-aks-wcus + + +Azure Monitor Workspace Location +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-eus-mac eastus +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-arc-wcus/providers/microsoft.monitor/accounts/ci-dev-arc-amw westcentralus +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.Monitor/accounts/ci-prod-aks-weu-mac westeurope +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/microsoft.monitor/accounts/ci-prod-arc-wcus westcentralus +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-amw/providers/microsoft.monitor/accounts/monitoring-metrics-amw-eus2euap eastus2euap +/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-amw/providers/microsoft.monitor/accounts/monitoring-metrics-amw-wcus westcentralus + + diff --git a/internal/alerts/example-alert-template.json b/internal/alerts/example-alert-template.json new file mode 100644 index 000000000..ce865c59d --- /dev/null +++ b/internal/alerts/example-alert-template.json @@ -0,0 +1,259 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "containerinsights_dev_mac_eus_alerts", + "type": "Microsoft.AlertsManagement/prometheusRuleGroups", + "apiVersion": "2023-03-01", + "location": "eastus", + "properties": { + "description": "rule group for cluster ci-dev-aks-mac-eus in MAC: ci-dev-aks-eus-mac", + "scopes": [ + "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-eus-mac", + "/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-mac-eus" + ], + "enabled": true, + "clusterName": "ci-dev-aks-mac-eus", + "rules": [ + { + "alert": "Amd64 metric missing in cluster ci-dev-aks-mac-eus", + "expression": "absent(node_uname_info{machine=\"x86_64\"}) == 1 or node_uname_info{machine=\"x86_64\"} == 0", + "for": "PT3M", + "annotations": { + "description": "Amd64 metric missing in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = node in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"node\"}) == 1 or up{job=\"node\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = node in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kubelet in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kubelet\"}) == 1 or up{job=\"kubelet\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kubelet in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = windows-exporter in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"windows-exporter\"}) == 1 or up{job=\"windows-exporter\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = windows-exporter in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kube-proxy in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kube-proxy\"}) == 1 or up{job=\"kube-proxy\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kube-proxy in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kube-apiserver in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kube-apiserver\"}) == 1 or up{job=\"kube-apiserver\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kube-apiserver in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kube-proxy-windows\"}) == 1 or up{job=\"kube-proxy-windows\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kube-state-metrics\"}) == 1 or up{job=\"kube-state-metrics\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = cadvisor in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"cadvisor\"}) == 1 or up{job=\"cadvisor\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = cadvisor in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "up metric missing for target = kube-dns in cluster ci-dev-aks-mac-eus", + "expression": "absent(up{job=\"kube-dns\"}) == 1 or up{job=\"kube-dns\"} == 0", + "for": "PT3M", + "annotations": { + "description": "up metric is not flowing for target = kube-dns in cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "CPU usage % greater than 90 for prometheus-collector containers on cluster ci-dev-aks-mac-eus", + "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"kube-system\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"kube-system\"}) )) by (container, pod) > 0.9", + "for": "PT3M", + "annotations": { + "description": "CPU usage greater than 90% for prometheus-collector on cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "CPU usage % greater than 50 for prometheus-collector containers on cluster ci-dev-aks-mac-eus", + "expression": "sum(sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\", namespace=\"kube-system\", container=\"prometheus-collector\"}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\", namespace=\"kube-system\"}) )) by (container, pod) > 0.5", + "for": "PT3M", + "annotations": { + "description": "CPU usage greater than 5% for prometheus-collector on cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + }, + { + "alert": "Memory usage is high for prometheus-collector containers on cluster ci-dev-aks-mac-eus", + "expression": "(sum(container_memory_working_set_bytes{namespace=\"kube-system\", container=\"prometheus-collector\", image!=\"\"}) by (container, pod) / sum(kube_pod_container_resource_requests{namespace=\"kube-system\", container=\"prometheus-collector\", resource=\"memory\"}) by (container, pod)) > 1.9", + "for": "PT3M", + "annotations": { + "description": "Memory usage is high for prometheus-collector containers on cluster ci-dev-aks-mac-eus" + }, + "severity": 4, + "resolveConfiguration": { + "autoResolved": true, + "timeToResolve": "PT10M" + }, + "actions": [ + { + "actionGroupId": "/subscriptions/13d371f9-5a39-46d5-8e1b-60158c49db84/resourceGroups/ContainerInsightsPrometheusCollector-Prod/providers/microsoft.insights/actiongroups/AMA-PrometheusCollectorAlertGroup" + } + ] + } + ] + } + } + ] +} diff --git a/internal/docs/ESRPCodeSign.md b/internal/docs/ESRPCodeSign.md index 51ca09e33..162344327 100644 --- a/internal/docs/ESRPCodeSign.md +++ b/internal/docs/ESRPCodeSign.md @@ -10,4 +10,4 @@ I have followed this [doc](https://eng.ms/docs/more/containers-secure-supply-cha For verification of signing we can do through 2 ways. 1. Locally through the doc https://eng.ms/docs/more/containers-secure-supply-chain/signing under validation section using notation. We have to use our own [certificate](https://ms.portal.azure.com/#view/Microsoft_Azure_KeyVault/ListObjectVersionsRBACBlade/~/overview/objectType/certificates/objectId/https%3A%2F%2Fesrpprometheuskv.vault.azure.net%2Fcertificates%2FESRPReqPrometheusCert/vaultResourceUri/%2Fsubscriptions%2F9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb%2FresourceGroups%2FESRPPrometheus%2Fproviders%2FMicrosoft.KeyVault%2Fvaults%2FESRPPrometheusKV/vaultId/%2Fsubscriptions%2F9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb%2FresourceGroups%2FESRPPrometheus%2Fproviders%2FMicrosoft.KeyVault%2Fvaults%2FESRPPrometheusKV) instead of the one in the example. -2. We can do a docker pull on the signed images and it will not have the following error message - "manifest verification failed for digest sha256..." +2. We can do a "docker manifest inspect -v " on the signed images and it will not have the following error message - "manifest verification failed for digest sha256..." diff --git a/internal/monitoring/README.md b/internal/monitoring/README.md index 87c9d0f0e..7c50ef28e 100644 --- a/internal/monitoring/README.md +++ b/internal/monitoring/README.md @@ -1,32 +1,29 @@ -### **This wiki contains links of all the resources related to alerts and dashboards of the CI CD clusters** +### **This wiki contains links of all the resources related to alerts and dashboards of the CI CD and prod monitoring near ring clusters** -**Links to all AKS clusters used in CI CD** +Below is the linking of the AKS cluster to Azure Monitor Workspace to Grafana for cicd and prod monitoring clusters: -Below are the links to all the AKS clusters used in CI CD. -We have 2 dev clusters and 2 prod clusters. +ci/cd clusters (cluster --> amw --> grafana) +============================================ -* ci-prod-aks-eus - [link](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-prod-aks-eus/overview) +[AKS] +dev=[ci-dev-aks-mac-eus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-mac-eus/overview) --> [ci-dev-aks-eus-mac](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/microsoft.monitor/accounts/ci-dev-aks-eus-mac/resourceOverviewId) --> [cicd-graf-metrics-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.Dashboard/grafana/cicd-graf-metrics-wcus/overview) -* ci-prod-aks-mac-weu - [link](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.ContainerService/managedClusters/ci-prod-aks-mac-weu/overview) +prod=[ci-prod-aks-mac-weu](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/Microsoft.ContainerService/managedClusters/ci-prod-aks-mac-weu/overview) --> [ci-prod-aks-weu-mac](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-aks-mac-weu-rg/providers/microsoft.monitor/accounts/ci-prod-aks-weu-mac/resourceOverviewId) --> [cicd-graf-metrics-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.Dashboard/grafana/cicd-graf-metrics-wcus/overview) -* ci-dev-aks-mac-eus - [link](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-mac-eus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-mac-eus/overview) +[ARC] +dev=[ci-dev-arc-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-arc-wcus/providers/Microsoft.ContainerService/managedClusters/ci-dev-arc-wcus/overview) --> [ci-dev-arc-amw](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-arc-wcus/providers/microsoft.monitor/accounts/ci-dev-arc-amw/resourceOverviewId) --> [cicd-graf-metrics-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.Dashboard/grafana/cicd-graf-metrics-wcus/overview) -* ci-dev-aks-wcus - [link](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-dev-aks-wcus-rg/providers/Microsoft.ContainerService/managedClusters/ci-dev-aks-wcus/overview) +prod=[ci-prod-arc-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.ContainerService/managedClusters/ci-prod-arc-wcus/overview)--> [ci-prod-arc-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/microsoft.monitor/accounts/ci-prod-arc-wcus/resourceOverviewId) --> [cicd-graf-metrics-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/ci-prod-arc-wcus/providers/Microsoft.Dashboard/grafana/cicd-graf-metrics-wcus/overview) -**Links of Grafana Instances** -* ci-prod-aks-eus - [link](https://ci-prod-aks-eus-graf-dueya5aadyd2ghef.eus.grafana.azure.com) +canary/prod monitoring clusters (cluster --> amw -->grafana) +=========================================================== -* ci-prod-aks-mac-weu - [link](https://ci-prod-aks-weu-graf-fffvdrhqgkg6dxgm.weu.grafana.azure.com) +[monitoring-metrics-prod-aks-eus2euap](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/Microsoft.ContainerService/managedClusters/monitoring-metrics-prod-aks-eus2euap/overview) --> [monitoring-metrics-amw-eus2euap](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-amw/providers/microsoft.monitor/accounts/monitoring-metrics-amw-eus2euap/resourceOverviewId) --> [monitoring-grafana-metrics-westus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/microsoft.dashboard/grafana/mon-graf-metric-westus/overview) +[monitoring-metrics-prod-aks-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/Microsoft.ContainerService/managedClusters/monitoring-metrics-prod-aks-wcus/overview) --> [monitoring-metrics-amw-wcus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-amw/providers/microsoft.monitor/accounts/monitoring-metrics-amw-wcus/resourceOverviewId) --> [monitoring-grafana-metrics-westus](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb/resourceGroups/monitoring-metrics-prod-aks/providers/microsoft.dashboard/grafana/mon-graf-metric-westus/overview) -* ci-dev-aks-mac-eus - [link](https://ci-dev-aks-eus-graf-aje3bpf7d3ctc9h6.eus.grafana.azure.com) -* ci-dev-aks-wcus - [link](https://ci-dev-aks-wcus-graf-acfcb0a6emefghgn.wcus.grafana.azure.com) +**Dashboard for CI CD and prod monitoring clusters** -**Dashboard fo CI CD alerts** -* ci-prod-aks-eus - [link](https://ci-prod-aks-eus-graf-dueya5aadyd2ghef.eus.grafana.azure.com/d/dmzAODN4k/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1) +* CICD - [link](https://cicd-graf-metrics-wcus-dkechtfecuadeuaw.wcus.grafana.azure.com/d/gp9556IVy/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1) -* ci-prod-aks-mac-weu - [link](https://ci-prod-aks-weu-graf-fffvdrhqgkg6dxgm.weu.grafana.azure.com/d/gp9556IVz/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1) - -* ci-dev-aks-mac-eus - [link](https://ci-dev-aks-eus-graf-aje3bpf7d3ctc9h6.eus.grafana.azure.com/d/gp9556IVz/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1&from=1667414623523&to=1667457823523) - -* ci-dev-aks-wcus - [link](https://ci-dev-aks-wcus-graf-acfcb0a6emefghgn.wcus.grafana.azure.com/d/dmzAODN4k/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1) +* Prod near ring - [link](https://mon-graf-metric-westus-f5hvdcaxc3hjdcdm.wus.grafana.azure.com/d/gp9556IVy/cpu-and-memory-utilization-k-s-m-replicaset-and-daemonset?orgId=1) diff --git a/internal/monitoring/dashboards/ci-dev-aks-mac-eus-db.json b/internal/monitoring/dashboards/ci-dev-aks-mac-eus-db.json deleted file mode 100644 index fe3bd4c72..000000000 --- a/internal/monitoring/dashboards/ci-dev-aks-mac-eus-db.json +++ /dev/null @@ -1,1631 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 58, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 34, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"} == 0\r\n\r\n", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Amd64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "{__name__=\"node_uname_info\", cluster=\"ci-dev-aks-mac-eus\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 32, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"aarch64\"} == 0\r\n\r\n", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Aarch64 metric missing ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00025, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.00025 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"})", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Replicaset CPU %", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.00023 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", - "format": "time_series", - "range": true, - "refId": "A" - } - ], - "title": "K-S-M Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0008, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.0008 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"}) ", - "refId": "A" - } - ], - "title": "Daemonset CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0074, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0062 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Daemonset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00001183, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.00000953 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"})", - "refId": "A" - } - ], - "title": "K-S-M CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00265, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.00225 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Replicaset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = cadvisor in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-dns\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-dns in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy-windows\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Up{cluster=\"ci-dev-aks-mac-eus\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-apiserver\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kubelet \"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kubelet in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 57 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"windows-exporter\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = windows-exporter in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 65 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-dev-aks-eus-mac" - }, - "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = node in cluster ci-dev-aks-mac-eus", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-12h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", - "uid": "gp9556IVz", - "version": 15, - "weekStart": "" -} \ No newline at end of file diff --git a/internal/monitoring/dashboards/ci-dev-aks-wcus-db.json b/internal/monitoring/dashboards/ci-dev-aks-wcus-db.json deleted file mode 100644 index 8852bc26d..000000000 --- a/internal/monitoring/dashboards/ci-dev-aks-wcus-db.json +++ /dev/null @@ -1,1634 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 58, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "{__name__=\"node_uname_info\", cluster=\"ci-dev-aks-wcus\", domainname=\"(none)\", instance=\"aks-lrnm-38217791-vmss000004\", job=\"node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-lrnm-38217791-vmss000004\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 32, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"x86_64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Amd64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 34, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-wcus\", machine=\"aarch64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "aarch64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00014, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.00014 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{ image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*kube-state-metrics.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"})", - "refId": "A" - } - ], - "title": "K-S-M CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.033, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.033 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Daemonset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0078, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0078 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"})", - "range": true, - "refId": "A" - } - ], - "title": "Daemonset CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.002, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.002 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*kube-state-metrics.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"})", - "range": true, - "refId": "A" - } - ], - "title": "K-S-M Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.103, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.103 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) - sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\",cluster=\"ci-dev-aks-wcus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"cpu\"})", - "refId": "A" - } - ], - "title": "Replicaset CPU %", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0157, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0157 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "(sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector.*\"}) - sum(container_memory_working_set_bytes{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"})) / sum(kube_pod_container_resource_limits{cluster=\"ci-dev-aks-wcus\", namespace=\"monitoring\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Replicaset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Up{cluster=\"ci-dev-aks-wcus\", instance=\"10.240.0.61:9153\", job=\"kube-dns\", pod=\"coredns-59b6bf8b4f-p7c67\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"node\"} == 0\r\n", - "format": "time_series", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = node", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-apiserver\"} == 0\r\n", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-apiserver ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy\"} == 0\r\n", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kubelet\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kubelet", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy-windows\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-proxy-windows\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy-windows", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"windows-exporter\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"windows-exporter\"} == 0\r\n\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = windows-exporter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "\r\nabsent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-state-metrics\"} == 0\r\n", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-state-metrics ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "expr": "absent(up{cluster=\"ci-dev-aks-wcus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"cadvisor\"} == 0\r\n", - "refId": "A" - } - ], - "title": "up metric missing for target = cadvisor", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 64 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PpvC3iv4k" - }, - "editorMode": "code", - "expr": "\r\nabsent(up{cluster=\"ci-dev-aks-wcus\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-dev-aks-wcus\", job=\"kube-dns\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-dns ", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", - "uid": "dmzAODN4k", - "version": 10, - "weekStart": "" -} \ No newline at end of file diff --git a/internal/monitoring/dashboards/ci-prod-aks-eus-db.json b/internal/monitoring/dashboards/ci-prod-aks-eus-db.json deleted file mode 100644 index 6adf58d64..000000000 --- a/internal/monitoring/dashboards/ci-prod-aks-eus-db.json +++ /dev/null @@ -1,1625 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 58, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 32, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"x86_64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Amd64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 34, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-eus\", machine=\"aarch64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "aarch64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.000238, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.000238 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{ image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*kube-state-metrics.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"})", - "range": true, - "refId": "A" - } - ], - "title": "K-S-M CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.04, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.04 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"})", - "range": true, - "refId": "A" - } - ], - "title": "Daemonset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0105, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0105 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"})", - "range": true, - "refId": "A" - } - ], - "title": "Daemonset CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0022, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0022 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*kube-state-metrics.*\"}) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"})", - "range": true, - "refId": "A" - } - ], - "title": "K-S-M Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.28, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.28 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) - sum(sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\",cluster=\"ci-prod-aks-eus\", pod=~\".*prometheus-collector-node.*\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"cpu\"})", - "range": true, - "refId": "A" - } - ], - "title": "Replicaset CPU %", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.023, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.023 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "(sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector.*\"}) - sum(container_memory_working_set_bytes{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\",container!=\"\", image!=\"\", pod=~\".*prometheus-collector-node.*\"})) / sum(kube_pod_container_resource_limits{cluster=\"ci-prod-aks-eus\", namespace=\"monitoring\", resource=\"memory\"})", - "range": true, - "refId": "A" - } - ], - "title": "Replicaset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Up{cluster=\"cimon-aks-wcus\", instance=\"10.240.0.61:9153\", job=\"kube-dns\", pod=\"coredns-59b6bf8b4f-p7c67\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"node\"} == 0\r\n", - "format": "time_series", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = node", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-apiserver\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-apiserver ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kubelet\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kubelet", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-prod-aks-eus\", job=\"kube-proxy-windows\"} == 0", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy-windows", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus-mdm" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus-mdm" - }, - "editorMode": "code", - "expr": " up{cluster=\"ci-prod-aks-eus\", job=\"windows-exporter\"} == 0 \r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = windows-exporter", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "\r\nabsent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-state-metrics\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-state-metrics ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "absent(up{cluster=\"ci-prod-aks-eus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"cadvisor\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = cadvisor", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 64 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PB5CBmv4z" - }, - "editorMode": "code", - "expr": "\r\nabsent(up{cluster=\"ci-prod-aks-eus\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-prod-aks-eus\", job=\"kube-dns\"} == 0\r\n", - "range": true, - "refId": "A" - } - ], - "title": "up metric missing for target = kube-dns ", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", - "uid": "dmzAODN4k", - "version": 45, - "weekStart": "" -} \ No newline at end of file diff --git a/internal/monitoring/dashboards/ci-prod-aks-mac-weu-db.json b/internal/monitoring/dashboards/ci-prod-aks-mac-weu-db.json deleted file mode 100644 index f5dd9758b..000000000 --- a/internal/monitoring/dashboards/ci-prod-aks-mac-weu-db.json +++ /dev/null @@ -1,1552 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 58, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 32, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Amd64 metric missing ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 34, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "editorMode": "code", - "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"aarch64\"} == 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "aarch64 metric missing", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0002, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0002 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"})", - "refId": "A" - } - ], - "title": "Replicaset CPU %", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00023, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.00023 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "K-S-M Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.00071 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"}) ", - "refId": "A" - } - ], - "title": "Daemonset CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.0074, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.0074 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Daemonset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00000953, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.00000953 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "editorMode": "code", - "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"})", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "K-S-M CPU%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 0.00265, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0.00265 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", - "refId": "A" - } - ], - "title": "Replicaset Memory%", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = cadvisor in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-dns\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-dns\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-dns in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy-windows\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy-windows\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy-windows in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-state-metrics in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-proxy in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-apiserver\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-apiserver\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kubelet\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kubelet\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = kubelet in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 57 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"windows-exporter\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"windows-exporter\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = windows-exporter in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 65 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "ci-prod-aks-weu-mac" - }, - "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"} == 0", - "refId": "A" - } - ], - "title": "up metric missing for target = node in cluster ci-prod-aks-mac-weu", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", - "uid": "gp9556IVz", - "version": 25, - "weekStart": "" -} \ No newline at end of file diff --git a/internal/monitoring/dashboards/cicd-db.json b/internal/monitoring/dashboards/cicd-db.json new file mode 100644 index 000000000..547914803 --- /dev/null +++ b/internal/monitoring/dashboards/cicd-db.json @@ -0,0 +1,8838 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 42, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 84, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 0.00095, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"ci-prod-arc-wcus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-arc-wcus\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"ci-prod-arc-wcus\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "absent(up{cluster=\"ci-prod-arc-wcus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-arc-wcus\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-arc-wcus\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "expr": "absent(up{cluster=\"ci-prod-arc-wcus\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-arc-wcus\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster ci-prod-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"ci-prod-arc-wcus\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-prod-arc-wcus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-prod-arc-wcus\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-prod-arc-wcus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-arc-wcus\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 90 + }, + "id": 132, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-arc-wcus" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "cluster = ci-prod-arc-wcus", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 35, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 109, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 110, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 111, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 112, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 113, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 114, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 115, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 116, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 118, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 120, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-arc-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 121, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 122, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"ci-dev-arc-wcus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-arc-wcus\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"ci-dev-arc-wcus\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 123, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "absent(up{cluster=\"ci-dev-arc-wcus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-arc-wcus\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 124, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 125, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 126, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 127, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 128, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-arc-wcus\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 129, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "expr": "absent(up{cluster=\"ci-dev-arc-wcus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-arc-wcus\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster ci-dev-arc-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"ci-dev-arc-wcus\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 130, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-dev-arc-wcus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-dev-arc-wcus\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 131, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-dev-arc-wcus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-arc-wcus\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 133, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-arc-amw" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "Cluster = ci-dev-arc-wcus", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 60, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 0.00095, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 86, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 90, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 91, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 92, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 93, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 94, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 95, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 96, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 97, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-prod-aks-mac-weu\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 98, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 99, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"ci-prod-aks-mac-weu\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 60 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 60 + }, + "id": 101, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 102, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 68 + }, + "id": 103, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 104, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 105, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-prod-aks-mac-weu\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 84 + }, + "id": 106, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "expr": "absent(up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"}) == 1 or up{cluster=\"ci-prod-aks-mac-weu\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"ci-prod-aks-mac-weu\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 84 + }, + "id": 107, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 92 + }, + "id": 108, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-prod-aks-mac-weu\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 92 + }, + "id": 134, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-prod-aks-weu-mac" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "Cluster = ci-prod-aks-mac-weu", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 85, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 0.00095, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"ci-dev-aks-mac-eus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"ci-dev-aks-mac-eus\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 61 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 61 + }, + "id": 76, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster ci-prod-aks-mac-weu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 69 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": " up{cluster=\"ci-dev-aks-mac-eus\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 81, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "expr": "absent(up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"}) == 1 or up{cluster=\"ci-dev-aks-mac-eus\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster ci-dev-aks-mac-eus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"ci-dev-aks-mac-eus\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 82, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 93 + }, + "id": 83, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"ci-dev-aks-mac-eus\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 93 + }, + "id": 135, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ci-dev-aks-eus-mac" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "Cluster = ci-dev-aks-mac-eus", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", + "uid": "gp9556IVy", + "version": 38, + "weekStart": "" + } diff --git a/internal/monitoring/dashboards/prod-near-ring-db.json b/internal/monitoring/dashboards/prod-near-ring-db.json new file mode 100644 index 000000000..a96f5f673 --- /dev/null +++ b/internal/monitoring/dashboards/prod-near-ring-db.json @@ -0,0 +1,4444 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 44, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 35, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 109, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 110, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 111, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 112, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 113, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 114, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 115, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 116, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 118, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 120, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-wcus\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 121, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 122, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"cadvisor\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"monitoring-metrics-prod-aks-wcus\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 123, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 124, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 125, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 126, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 127, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 128, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 129, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"node\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-wcus\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster monitoring-metrics-prod-aks-wcus", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"monitoring-metrics-prod-aks-wcus\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 130, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"monitoring-metrics-prod-aks-wcus\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"monitoring-metrics-prod-aks-wcus\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 131, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"monitoring-metrics-prod-aks-wcus\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"monitoring-metrics-prod-aks-wcus\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 90 + }, + "id": 132, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-wcus" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "Cluster = monitoring-metrics-prod-aks-wcus", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 84, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "max": 0.00095, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "(sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"}))/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"cpu\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", pod=~\"ama-metrics-node.*\"}) - sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", pod=~\"ama-metrics-ksm.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"cpu\"}) ", + "refId": "A" + } + ], + "title": "Daemonset CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-node.*\"}) ", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})/ sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"cpu\"})", + "refId": "A" + } + ], + "title": "K-S-M CPU%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum ( node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", container!=\"\", pod=~\"ama-metrics-ksm.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.00023 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"memory\"})", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"}) ", + "format": "time_series", + "range": true, + "refId": "A" + } + ], + "title": "K-S-M Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Daemonset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"})", + "range": true, + "refId": "A" + } + ], + "title": "Daemonset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\", resource=\"memory\"})", + "refId": "A" + } + ], + "title": "Replicaset Memory%", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "(sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-node.*\"}) - sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", namespace=\"kube-system\",container!=\"\", image!=\"\", pod=~\"ama-metrics-ksm.*\"})) ", + "range": true, + "refId": "A" + } + ], + "title": "Replicaset Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-dns\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-dns in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"cadvisor\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"cadvisor\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = cadvisor in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", instance=\"ama-metrics-ksm.kube-system.svc.cluster.local:8080\", job=\"kube-state-metrics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-state-metrics\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-state-metrics\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = kube-state-metrics in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-proxy-windows\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy-windows in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-apiserver\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-apiserver in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kube-proxy\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kube-proxy in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"windows-exporter\"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = windows-exporter in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": " up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"kubelet \"} == 0", + "range": true, + "refId": "A" + } + ], + "title": "up metric missing for target = kubelet in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "expr": "absent(up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"node\"}) == 1 or up{cluster=\"monitoring-metrics-prod-aks-eus2euap\", job=\"node\"} == 0", + "refId": "A" + } + ], + "title": "up metric missing for target = node in cluster monitoring-metrics-amw-eus2euap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"node_uname_info\", cluster=\"monitoring-metrics-prod-aks-eus2euap\", domainname=\"(none)\", instance=\"10.240.0.115:9100\", job=\"custom-scrape-job-node\", machine=\"x86_64\", metrics_path=\"/metrics\", nodename=\"aks-agentpool-86683822-vmss000001\", release=\"5.4.0-1091-azure\", sysname=\"linux\", version=\"#96~18.04.1-ubuntu smp tue aug 30 19:15:32 utc 2022\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"monitoring-metrics-prod-aks-eus2euap\", machine=\"aarch64\"}) == 1 or node_uname_info{cluster=\"monitoring-metrics-prod-aks-eus2euap\", machine=\"aarch64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Aarch64 metric missing ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "absent(node_uname_info{cluster=\"monitoring-metrics-prod-aks-eus2euap\", machine=\"x86_64\"}) == 1 or node_uname_info{cluster=\"monitoring-metrics-prod-aks-eus2euap\", machine=\"x86_64\"} == 0\r\n\r\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Amd64 metric missing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 133, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "monitoring-metrics-amw-eus2euap" + }, + "editorMode": "code", + "expr": "count (alerts{alertstate=\"firing\"}) by (alertname, cluster)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Firing alerts", + "type": "timeseries" + } + ], + "title": "cluster = monitoring-metrics-amw-eus2euap", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "CPU and Memory utilization k-s-m, replicaset and daemonset", + "uid": "gp9556IVy", + "version": 13, + "weekStart": "" + } diff --git a/mixins/kubernetes/rules/recording_and_alerting_rules/templates/ci_recommended_alerts.json b/mixins/kubernetes/rules/recording_and_alerting_rules/templates/ci_recommended_alerts.json index 13cdb3027..6143de054 100644 --- a/mixins/kubernetes/rules/recording_and_alerting_rules/templates/ci_recommended_alerts.json +++ b/mixins/kubernetes/rules/recording_and_alerting_rules/templates/ci_recommended_alerts.json @@ -8,6 +8,12 @@ "description": "Cluster name" } }, + "clusterResourceId": { + "type": "string", + "metadata": { + "description": "Cluster Resource Id" + } + }, "actionGroupResourceId": { "type": "string", "metadata": { @@ -40,7 +46,8 @@ "properties": { "description": "[concat(variables('kubernetesAlertRuleGroupDescription'), variables('version'))]", "scopes": [ - "[parameters('azureMonitorWorkspaceResourceId')]" + "[parameters('azureMonitorWorkspaceResourceId')]", + "[parameters('clusterResourceId')]" ], "clusterName": "[parameters('clusterName')]", "enabled": true, diff --git a/otelcollector/build/windows/Dockerfile b/otelcollector/build/windows/Dockerfile index 8983c24f7..17b813b3a 100644 --- a/otelcollector/build/windows/Dockerfile +++ b/otelcollector/build/windows/Dockerfile @@ -21,7 +21,7 @@ COPY ./configmapparser/*.rb $tmpdir/microsoft/configmapparser/ COPY ./configmapparser/default-prom-configs/*.yml $tmpdir/microsoft/otelcollector/default-prom-configs/ COPY ./opentelemetry-collector-builder/otelcollector.exe ./opentelemetry-collector-builder/collector-config-default.yml ./opentelemetry-collector-builder/collector-config-template.yml $tmpdir/microsoft/otelcollector/ COPY ./prom-config-validator-builder/promconfigvalidator.exe $tmpdir/ -COPY ./metricextension/me.config ./metricextension/me_internal.config ./metricextension/me_ds.config ./metricextension/me_ds_internal.config $tmpdir/metricextension/ +COPY ./metricextension/me.config ./metricextension/me_internal.config ./metricextension/me_ds.config ./metricextension/me_ds_win.config ./metricextension/me_ds_internal.config ./metricextension/me_ds_internal_win.config $tmpdir/metricextension/ COPY ./telegraf/telegraf-prometheus-collector-windows.conf $tmpdir/telegraf/ COPY ./fluent-bit/fluent-bit-windows.conf $tmpdir/fluent-bit/ COPY ./fluent-bit/fluent-bit-parsers.conf $tmpdir/fluent-bit/ diff --git a/otelcollector/build/windows/scripts/main.ps1 b/otelcollector/build/windows/scripts/main.ps1 index 0ab6f14e2..c8b4f22da 100644 --- a/otelcollector/build/windows/scripts/main.ps1 +++ b/otelcollector/build/windows/scripts/main.ps1 @@ -1,5 +1,5 @@ #setting it to replicaset by default -$me_config_file = '/opt/metricextension/me_ds.config' +$me_config_file = '/opt/metricextension/me_ds_win.config' function Set-EnvironmentVariablesAndConfigParser { # Set unfair semaphore wait for better initial CPU performance @@ -275,10 +275,10 @@ function Set-EnvironmentVariablesAndConfigParser { } else { if ($cluster_override -eq "true") { - $meConfigFile = "/opt/metricextension/me_ds_internal.config" + $meConfigFile = "/opt/metricextension/me_ds_internal_win.config" } else { - $meConfigFile = "/opt/metricextension/me_ds.config" + $meConfigFile = "/opt/metricextension/me_ds_win.config" } } [System.Environment]::SetEnvironmentVariable("ME_CONFIG_FILE", $meConfigFile, "Process") @@ -406,7 +406,7 @@ function Start-ME { } else { Start-Process -NoNewWindow -FilePath "/opt/metricextension/MetricsExtension/MetricsExtension.Native.exe" -ArgumentList @("-Logger", "File", "-LogLevel", "Debug", "-LocalControlChannel", "-TokenSource", "AMCS", "-DataDirectory", "C:\opt\genevamonitoringagent\datadirectory\mcs\metricsextension\", "-Input", "otlp_grpc_prom", "-ConfigOverridesFilePath", $me_config_file) > $null - # /opt/metricextension/MetricsExtension/MetricsExtension.Native.exe -Logger Console -LogLevel Info -LocalControlChannel -TokenSource AMCS -DataDirectory C:\opt\genevamonitoringagent\datadirectory\mcs\metricsextension\ -Input otlp_grpc_prom -ConfigOverridesFilePath '/opt/metricextension/me_ds.config' + # /opt/metricextension/MetricsExtension/MetricsExtension.Native.exe -Logger Console -LogLevel Info -LocalControlChannel -TokenSource AMCS -DataDirectory C:\opt\genevamonitoringagent\datadirectory\mcs\metricsextension\ -Input otlp_grpc_prom -ConfigOverridesFilePath '/opt/metricextension/me_ds_win.config' } } else { diff --git a/otelcollector/configmapparser/prometheus-config-merger.rb b/otelcollector/configmapparser/prometheus-config-merger.rb index c0c9c6808..ee353aa37 100644 --- a/otelcollector/configmapparser/prometheus-config-merger.rb +++ b/otelcollector/configmapparser/prometheus-config-merger.rb @@ -365,14 +365,6 @@ def populateDefaultPrometheusConfig contents = contents.gsub("$$NODE_NAME$$", ENV["NODE_NAME"]) File.open(@windowsexporterDefaultDsFile, "w") { |file| file.puts contents } defaultConfigs.push(@windowsexporterDefaultDsFile) - - # If advanced mode is enabled, but not the windows daemonset, scrape windows kubelet from the replicaset as if it's simple mode - elsif currentControllerType == @replicasetControllerType && advancedMode == true && windowsDaemonset == false && ENV["OS_TYPE"].downcase == "linux" - UpdateScrapeIntervalConfig(@windowsexporterDefaultRsSimpleFile, windowsexporterScrapeInterval) - if !winexporterMetricsKeepListRegex.nil? && !winexporterMetricsKeepListRegex.empty? - AppendMetricRelabelConfig(@windowsexporterDefaultRsSimpleFile, winexporterMetricsKeepListRegex) - end - defaultConfigs.push(@windowsexporterDefaultRsSimpleFile) end end @@ -399,14 +391,6 @@ def populateDefaultPrometheusConfig contents = contents.gsub("$$NODE_NAME$$", ENV["NODE_NAME"]) File.open(@windowskubeproxyDefaultDsFile, "w") { |file| file.puts contents } defaultConfigs.push(@windowskubeproxyDefaultDsFile) - - # If advanced mode is enabled, but not the windows daemonset, scrape windows kubelet from the replicaset as if it's simple mode - elsif currentControllerType == @replicasetControllerType && advancedMode == true && windowsDaemonset == false && ENV["OS_TYPE"].downcase == "linux" - UpdateScrapeIntervalConfig(@windowskubeproxyDefaultFileRsSimpleFile, windowskubeproxyScrapeInterval) - if !winkubeproxyMetricsKeepListRegex.nil? && !winkubeproxyMetricsKeepListRegex.empty? - AppendMetricRelabelConfig(@windowskubeproxyDefaultFileRsSimpleFile, winkubeproxyMetricsKeepListRegex) - end - defaultConfigs.push(@windowskubeproxyDefaultFileRsSimpleFile) end end diff --git a/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-daemonset.yaml b/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-daemonset.yaml index d039a0100..25bacebf5 100644 --- a/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-daemonset.yaml +++ b/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-daemonset.yaml @@ -128,7 +128,7 @@ spec: - name: MODE value: "advanced" # only supported mode is 'advanced', any other value will be the default/non-advance mode - name: WINMODE - value: "" # WINDOWS: only supported mode is 'advanced', any other value will be the default/non-advance mode + value: "advanced" # WINDOWS: only supported mode is 'advanced', any other value will be the default/non-advance mode - name: MINIMAL_INGESTION_PROFILE value: "true" # only supported value is the string "true" securityContext: diff --git a/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-deployment.yaml b/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-deployment.yaml index 532f5c930..31c36855b 100644 --- a/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-deployment.yaml +++ b/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/templates/ama-metrics-deployment.yaml @@ -139,7 +139,7 @@ spec: - name: MODE value: "advanced" # only supported mode is 'advanced', any other value will be the default/non-advance mode - name: WINMODE - value: "" # WINDOWS: only supported mode is 'advanced', any other value will be the default/non-advance mode + value: "advanced" # WINDOWS: only supported mode is 'advanced', any other value will be the default/non-advance mode - name: MINIMAL_INGESTION_PROFILE value: "true" # only supported value is the string "true" securityContext: diff --git a/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/values-template.yaml b/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/values-template.yaml index e9cfcef4e..1ffa20355 100644 --- a/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/values-template.yaml +++ b/otelcollector/deploy/addon-chart/azure-monitor-metrics-addon/values-template.yaml @@ -44,7 +44,7 @@ AzureMonitorMetrics: # ImageTag: https://msazure.visualstudio.com/CloudNativeCompute/_git/aks-rp?path=/ccp/charts/kube-control-plane/templates/_images.tpl&version=GBrashmi/prom-addon-arm64&line=530&lineEnd=530&lineStartColumn=28&lineEndColumn=53&lineStyle=plain&_a=contents AddonTokenAdapter: ImageRepository: "/aks/msi/addon-token-adapter" - ImageTag: "master.221118.2" + ImageTag: "master.230804.1" ImageRepositoryWin: "/aks/hcp/addon-token-adapter" ImageTagWin: "20230120winbeta" ArcExtension: ${ARC_EXTENSION} diff --git a/otelcollector/fluent-bit/src/telemetry.go b/otelcollector/fluent-bit/src/telemetry.go index bbc9b94cc..6274d6278 100644 --- a/otelcollector/fluent-bit/src/telemetry.go +++ b/otelcollector/fluent-bit/src/telemetry.go @@ -14,8 +14,8 @@ import ( "regexp" "strconv" "strings" - "time" "sync" + "time" "github.com/fluent/fluent-bit-go/output" "github.com/microsoft/ApplicationInsights-Go/appinsights" @@ -27,24 +27,22 @@ import ( ) type meMetricsProcessedCount struct { - DimBytesProcessedCount float64 - DimBytesSentToPubCount float64 - DimMetricsSentToPubCount float64 - Value float64 + DimBytesProcessedCount float64 + DimBytesSentToPubCount float64 + DimMetricsSentToPubCount float64 + Value float64 } type meMetricsReceivedCount struct { - Value float64 + Value float64 } - - var ( // CommonProperties indicates the dimensions that are sent with every event/metric CommonProperties map[string]string // TelemetryClient is the client used to send the telemetry TelemetryClient appinsights.TelemetryClient - // Invalid Prometheus config validation environemnt variable used for telemetry + // Invalid Prometheus config validation environment variable used for telemetry InvalidCustomPrometheusConfig string // Default Collector config DefaultPrometheusConfig string @@ -66,6 +64,34 @@ var ( WinExporterKeepListRegex string // Windows KubeProxy metrics keep list regex WinKubeProxyKeepListRegex string + // Pod Annotation metrics keep list regex + PodannotationKeepListRegex string + // Kappie Basic metrics keep list regex + KappieBasicKeepListRegex string + // Kubelet scrape interval + KubeletScrapeInterval string + // CoreDNS scrape interval + CoreDNSScrapeInterval string + // CAdvisor scrape interval + CAdvisorScrapeInterval string + // KubeProxy scrape interval + KubeProxyScrapeInterval string + // API Server scrape interval + ApiServerScrapeInterval string + // KubeState scrape interval + KubeStateScrapeInterval string + // Node Exporter scrape interval + NodeExporterScrapeInterval string + // Windows Exporter scrape interval + WinExporterScrapeInterval string + // Windows KubeProxy scrape interval + WinKubeProxyScrapeInterval string + // PrometheusCollector Health scrape interval + PromHealthScrapeInterval string + // Pod Annotation scrape interval + PodAnnotationScrapeInterval string + // Kappie Basic scrape interval + KappieBasicScrapeInterval string // meMetricsProcessedCount map, which holds references to metrics per metric account meMetricsProcessedCountMap = make(map[string]*meMetricsProcessedCount) // meMetricsProcessedCountMapMutex -- used for reading & writing locks on meMetricsProcessedCountMap @@ -79,7 +105,7 @@ var ( const ( coresAttachedTelemetryIntervalSeconds = 600 ksmAttachedTelemetryIntervalSeconds = 600 - meMetricsTelemetryIntervalSeconds = 300 + meMetricsTelemetryIntervalSeconds = 300 coresAttachedTelemetryName = "ClusterCoreCapacity" linuxCpuCapacityTelemetryName = "LiCapacity" linuxNodeCountTelemetryName = "LiNodeCnt" @@ -118,6 +144,7 @@ const ( fluentbitExportingFailedTag = "prometheus.log.exportingfailed" fluentbitFailedScrapeTag = "prometheus.log.failedscrape" keepListRegexHashFilePath = "/opt/microsoft/configmapparser/config_def_targets_metrics_keep_list_hash" + intervalHashFilePath = "/opt/microsoft/configmapparser/config_def_targets_scrape_intervals_hash" amcsConfigFilePath = "/etc/mdsd.d/config-cache/metricsextension/TokenConfig.json" ) @@ -261,6 +288,34 @@ func InitializeTelemetryClient(agentVersion string) (int, error) { NodeExporterKeepListRegex = regexHash["NODEEXPORTER_METRICS_KEEP_LIST_REGEX"] WinExporterKeepListRegex = regexHash["WINDOWSEXPORTER_METRICS_KEEP_LIST_REGEX"] WinKubeProxyKeepListRegex = regexHash["WINDOWSKUBEPROXY_METRICS_KEEP_LIST_REGEX"] + PodannotationKeepListRegex = regexHash["POD_ANNOTATION_METRICS_KEEP_LIST_REGEX"] + KappieBasicKeepListRegex = regexHash["KAPPIEBASIC_METRICS_KEEP_LIST_REGEX"] + } + } + + // Reading scrape interval hash file for telemetry + intervalFileContents, err := ioutil.ReadFile(intervalHashFilePath) + if err != nil { + Log("Error while opening interval hash file - %v\n", err) + } else { + Log("Successfully read interval hash file contents for telemetry\n") + var intervalHash map[string]string + err = yaml.Unmarshal([]byte(intervalFileContents), &intervalHash) + if err != nil { + Log("Error while unmarshalling interval hash file - %v\n", err) + } else { + KubeletScrapeInterval = intervalHash["KUBELET_SCRAPE_INTERVAL"] + CoreDNSScrapeInterval = intervalHash["COREDNS_SCRAPE_INTERVAL"] + CAdvisorScrapeInterval = intervalHash["CADVISOR_SCRAPE_INTERVAL"] + KubeProxyScrapeInterval = intervalHash["KUBEPROXY_SCRAPE_INTERVAL"] + ApiServerScrapeInterval = intervalHash["APISERVER_SCRAPE_INTERVAL"] + KubeStateScrapeInterval = intervalHash["KUBESTATE_SCRAPE_INTERVAL"] + NodeExporterScrapeInterval = intervalHash["NODEEXPORTER_SCRAPE_INTERVAL"] + WinExporterScrapeInterval = intervalHash["WINDOWSEXPORTER_SCRAPE_INTERVAL"] + WinKubeProxyScrapeInterval = intervalHash["WINDOWSKUBEPROXY_SCRAPE_INTERVAL"] + PromHealthScrapeInterval = intervalHash["PROMETHEUS_COLLECTOR_HEALTH_SCRAPE_INTERVAL"] + PodAnnotationScrapeInterval = intervalHash["POD_ANNOTATION_SCRAPE_INTERVAL"] + KappieBasicScrapeInterval = intervalHash["KAPPIEBASIC_SCRAPE_INTERVAL"] } } @@ -280,12 +335,12 @@ func SendCoreCountToAppInsightsMetrics() { coreCountTelemetryTicker := time.NewTicker(time.Second * time.Duration(coresAttachedTelemetryIntervalSeconds)) for ; true; <-coreCountTelemetryTicker.C { - telemetryProperties := map[string]int64 { + telemetryProperties := map[string]int64{ windowsCpuCapacityTelemetryName: 0, - windowsNodeCountTelemetryName: 0, - virtualNodeCountTelemetryName: 0, - arm64CpuCapacityTelemetryName: 0, - arm64NodeCountTelemetryName: 0, + windowsNodeCountTelemetryName: 0, + virtualNodeCountTelemetryName: 0, + arm64CpuCapacityTelemetryName: 0, + arm64NodeCountTelemetryName: 0, } nodeList, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) @@ -302,10 +357,10 @@ func SendCoreCountToAppInsightsMetrics() { if node.Labels == nil { SendException(fmt.Sprintf("Labels are missing for the node: %s when getting core capacity", node.Name)) } else if node.Labels["type"] == "virtual-kubelet" { - // Do not add core capacity total for virtual nodes as this could be extremely large - // Just count how many virtual nodes exist - telemetryProperties[virtualNodeCountTelemetryName] += 1 - continue + // Do not add core capacity total for virtual nodes as this could be extremely large + // Just count how many virtual nodes exist + telemetryProperties[virtualNodeCountTelemetryName] += 1 + continue } else { osLabel = node.Labels["kubernetes.io/os"] archLabel = node.Labels["kubernetes.io/arch"] @@ -493,21 +548,21 @@ func UpdateMEMetricsProcessedCount(records []map[interface{}]interface{}) int { if err == nil { metricsAccountName := groupMatches[3] - + bytesProcessedCount, e := strconv.ParseFloat(groupMatches[5], 64) - if e != nil{ + if e != nil { bytesProcessedCount = 0.0 } - - metricsSentToPubCount,e := strconv.ParseFloat(groupMatches[6], 64) + + metricsSentToPubCount, e := strconv.ParseFloat(groupMatches[6], 64) if e != nil { metricsSentToPubCount = 0.0 } - bytesSentToPubCount,e := strconv.ParseFloat(groupMatches[7], 64) + bytesSentToPubCount, e := strconv.ParseFloat(groupMatches[7], 64) if e != nil { bytesSentToPubCount = 0.0 } - + //update map meMetricsProcessedCountMapMutex.Lock() @@ -520,12 +575,12 @@ func UpdateMEMetricsProcessedCount(records []map[interface{}]interface{}) int { ref.Value += metricsProcessedCount } else { - m := &meMetricsProcessedCount { - DimBytesProcessedCount: bytesProcessedCount, - DimBytesSentToPubCount: bytesSentToPubCount, - DimMetricsSentToPubCount: metricsSentToPubCount, - Value: metricsProcessedCount, - } + m := &meMetricsProcessedCount{ + DimBytesProcessedCount: bytesProcessedCount, + DimBytesSentToPubCount: bytesSentToPubCount, + DimMetricsSentToPubCount: metricsSentToPubCount, + Value: metricsProcessedCount, + } meMetricsProcessedCountMap[metricsAccountName] = m } meMetricsProcessedCountMapMutex.Unlock() @@ -562,12 +617,12 @@ func PushMEProcessedAndReceivedCountToAppInsightsMetrics() { for ; true; <-ticker.C { meMetricsProcessedCountMapMutex.Lock() - for k,v := range meMetricsProcessedCountMap { + for k, v := range meMetricsProcessedCountMap { metric := appinsights.NewMetricTelemetry("meMetricsProcessedCount", v.Value) metric.Properties["metricsAccountName"] = k - metric.Properties["bytesProcessedCount"] = fmt.Sprintf("%.2f",v.DimBytesProcessedCount) - metric.Properties["metricsSentToPubCount"] = fmt.Sprintf("%.2f",v.DimMetricsSentToPubCount) - metric.Properties["bytesSentToPubCount"] = fmt.Sprintf("%.2f",v.DimBytesSentToPubCount) + metric.Properties["bytesProcessedCount"] = fmt.Sprintf("%.2f", v.DimBytesProcessedCount) + metric.Properties["metricsSentToPubCount"] = fmt.Sprintf("%.2f", v.DimMetricsSentToPubCount) + metric.Properties["bytesSentToPubCount"] = fmt.Sprintf("%.2f", v.DimBytesSentToPubCount) if InvalidCustomPrometheusConfig != "" { metric.Properties["InvalidCustomPrometheusConfig"] = InvalidCustomPrometheusConfig @@ -602,7 +657,49 @@ func PushMEProcessedAndReceivedCountToAppInsightsMetrics() { if WinKubeProxyKeepListRegex != "" { metric.Properties["WinKubeProxyKeepListRegex"] = WinKubeProxyKeepListRegex } - + if PodannotationKeepListRegex != "" { + metric.Properties["PodannotationKeepListRegex"] = PodannotationKeepListRegex + } + if KappieBasicKeepListRegex != "" { + metric.Properties["KappieBasicKeepListRegex"] = KappieBasicKeepListRegex + } + if KubeletScrapeInterval != "" { + metric.Properties["KubeletScrapeInterval"] = KubeletScrapeInterval + } + if CoreDNSScrapeInterval != "" { + metric.Properties["CoreDNSScrapeInterval"] = CoreDNSScrapeInterval + } + if CAdvisorScrapeInterval != "" { + metric.Properties["CAdvisorScrapeInterval"] = CAdvisorScrapeInterval + } + if KubeProxyScrapeInterval != "" { + metric.Properties["KubeProxyScrapeInterval"] = KubeProxyScrapeInterval + } + if ApiServerScrapeInterval != "" { + metric.Properties["ApiServerScrapeInterval"] = ApiServerScrapeInterval + } + if KubeStateScrapeInterval != "" { + metric.Properties["KubeStateScrapeInterval"] = KubeStateScrapeInterval + } + if NodeExporterScrapeInterval != "" { + metric.Properties["NodeExporterScrapeInterval"] = NodeExporterScrapeInterval + } + if WinExporterScrapeInterval != "" { + metric.Properties["WinExporterScrapeInterval"] = WinExporterScrapeInterval + } + if WinKubeProxyScrapeInterval != "" { + metric.Properties["WinKubeProxyScrapeInterval"] = WinKubeProxyScrapeInterval + } + if PromHealthScrapeInterval != "" { + metric.Properties["PromHealthScrapeInterval"] = PromHealthScrapeInterval + } + if PodAnnotationScrapeInterval != "" { + metric.Properties["PodAnnotationScrapeInterval"] = PodAnnotationScrapeInterval + } + if KappieBasicScrapeInterval != "" { + metric.Properties["KappieBasicScrapeInterval"] = KappieBasicScrapeInterval + } + TelemetryClient.Track(metric) } @@ -667,13 +764,13 @@ func UpdateMEReceivedMetricsCount(records []map[interface{}]interface{}) int { ref.Value += metricsReceivedCount } else { - m := &meMetricsReceivedCount { - Value: metricsReceivedCount, - } + m := &meMetricsReceivedCount{ + Value: metricsReceivedCount, + } meMetricsReceivedCountMap["na"] = m } meMetricsReceivedCountMapMutex.Unlock() - + // Add to the total that PublishTimeseriesVolume() uses if strings.ToLower(os.Getenv(envPrometheusCollectorHealth)) == "true" { TimeseriesVolumeMutex.Lock() diff --git a/otelcollector/metricextension/me_ds.config b/otelcollector/metricextension/me_ds.config index d2679d4f3..30645f994 100644 --- a/otelcollector/metricextension/me_ds.config +++ b/otelcollector/metricextension/me_ds.config @@ -9,6 +9,7 @@ "instance" ], "honorResourceAttributes":true, + "disableExemplars":true, "maxReceiveMessageSizeMBytes": 12 }, "publicationIntervalInSec":20, diff --git a/otelcollector/metricextension/me_ds_internal.config b/otelcollector/metricextension/me_ds_internal.config index d7ef23412..210db43fa 100644 --- a/otelcollector/metricextension/me_ds_internal.config +++ b/otelcollector/metricextension/me_ds_internal.config @@ -9,6 +9,7 @@ "instance" ], "honorResourceAttributes": false, + "disableExemplars":true, "maxReceiveMessageSizeMBytes": 12 }, "publicationIntervalInSec":20, diff --git a/otelcollector/metricextension/me_ds_internal_win.config b/otelcollector/metricextension/me_ds_internal_win.config new file mode 100644 index 000000000..d7ef23412 --- /dev/null +++ b/otelcollector/metricextension/me_ds_internal_win.config @@ -0,0 +1,26 @@ +{ + "otlp":{ + "endpoints":[ + "127.0.0.1:55680" + ], + "resourceAttributes":[ + "cluster", + "job", + "instance" + ], + "honorResourceAttributes": false, + "maxReceiveMessageSizeMBytes": 12 + }, + "publicationIntervalInSec":20, + "maxPublicationAttemptsPerMinute":12, + "maxPublicationPackageSizeInBytes":5000000, + "maxPublicationBytesPerMinute":200000000, + "maxPublicationMetricsPerMinute":10000000, + "maxAggregationQueueSize":7000000, + "maxNumberOfRawEventsPerCycle":5000000, + "compressMetricData":true, + "maxStringInternCacheSizeMb":5000, + "interningSwapPeriodInMin":10000, + "internalQueueSizeManagementPeriodInSec":10000, + "proxyDefinitionMode":1 +} diff --git a/otelcollector/metricextension/me_ds_win.config b/otelcollector/metricextension/me_ds_win.config new file mode 100644 index 000000000..d2679d4f3 --- /dev/null +++ b/otelcollector/metricextension/me_ds_win.config @@ -0,0 +1,26 @@ +{ + "otlp":{ + "endpoints":[ + "127.0.0.1:55680" + ], + "resourceAttributes":[ + "cluster", + "job", + "instance" + ], + "honorResourceAttributes":true, + "maxReceiveMessageSizeMBytes": 12 + }, + "publicationIntervalInSec":20, + "maxPublicationAttemptsPerMinute":12, + "maxPublicationPackageSizeInBytes":5000000, + "maxPublicationBytesPerMinute":200000000, + "maxPublicationMetricsPerMinute":10000000, + "maxAggregationQueueSize":7000000, + "maxNumberOfRawEventsPerCycle":5000000, + "compressMetricData":true, + "maxStringInternCacheSizeMb":5000, + "interningSwapPeriodInMin":10000, + "internalQueueSizeManagementPeriodInSec":10000, + "proxyDefinitionMode":1 +} diff --git a/otelcollector/metricextension/me_internal.config b/otelcollector/metricextension/me_internal.config index d7ef23412..210db43fa 100644 --- a/otelcollector/metricextension/me_internal.config +++ b/otelcollector/metricextension/me_internal.config @@ -9,6 +9,7 @@ "instance" ], "honorResourceAttributes": false, + "disableExemplars":true, "maxReceiveMessageSizeMBytes": 12 }, "publicationIntervalInSec":20, diff --git a/otelcollector/prom-config-validator-builder/main.go b/otelcollector/prom-config-validator-builder/main.go index d734901d1..acf8c1a92 100644 --- a/otelcollector/prom-config-validator-builder/main.go +++ b/otelcollector/prom-config-validator-builder/main.go @@ -121,6 +121,9 @@ func generateOtelConfig(promFilePath string, outputFilePath string, otelConfigTe regexString := relabelConfig["regex"].(string) modifiedRegexString := strings.ReplaceAll(regexString, "$$", "$") modifiedRegexString = strings.ReplaceAll(modifiedRegexString, "$", "$$") + // Doing the below since we dont want to substitute $ with $$ for env variables NODE_NAME and NODE_IP. + modifiedRegexString = strings.ReplaceAll(modifiedRegexString, "$$NODE_NAME", "$NODE_NAME") + modifiedRegexString = strings.ReplaceAll(modifiedRegexString, "$$NODE_IP", "$NODE_IP") relabelConfig["regex"] = modifiedRegexString } } @@ -129,6 +132,8 @@ func generateOtelConfig(promFilePath string, outputFilePath string, otelConfigTe replacement := relabelConfig["replacement"].(string) modifiedReplacementString := strings.ReplaceAll(replacement, "$$", "$") modifiedReplacementString = strings.ReplaceAll(modifiedReplacementString, "$", "$$") + modifiedReplacementString = strings.ReplaceAll(modifiedReplacementString, "$$NODE_NAME", "$NODE_NAME") + modifiedReplacementString = strings.ReplaceAll(modifiedReplacementString, "$$NODE_IP", "$NODE_IP") relabelConfig["replacement"] = modifiedReplacementString } } @@ -145,6 +150,8 @@ func generateOtelConfig(promFilePath string, outputFilePath string, otelConfigTe regexString := metricRelabelConfig["regex"].(string) modifiedRegexString := strings.ReplaceAll(regexString, "$$", "$") modifiedRegexString = strings.ReplaceAll(modifiedRegexString, "$", "$$") + modifiedRegexString = strings.ReplaceAll(modifiedRegexString, "$$NODE_NAME", "$NODE_NAME") + modifiedRegexString = strings.ReplaceAll(modifiedRegexString, "$$NODE_IP", "$NODE_IP") metricRelabelConfig["regex"] = modifiedRegexString } } @@ -154,6 +161,8 @@ func generateOtelConfig(promFilePath string, outputFilePath string, otelConfigTe replacement := metricRelabelConfig["replacement"].(string) modifiedReplacementString := strings.ReplaceAll(replacement, "$$", "$") modifiedReplacementString = strings.ReplaceAll(modifiedReplacementString, "$", "$$") + modifiedReplacementString = strings.ReplaceAll(modifiedReplacementString, "$$NODE_NAME", "$NODE_NAME") + modifiedReplacementString = strings.ReplaceAll(modifiedReplacementString, "$$NODE_IP", "$NODE_IP") metricRelabelConfig["replacement"] = modifiedReplacementString } } diff --git a/otelcollector/scripts/setup.sh b/otelcollector/scripts/setup.sh index 47ae73444..755b59636 100644 --- a/otelcollector/scripts/setup.sh +++ b/otelcollector/scripts/setup.sh @@ -44,7 +44,7 @@ echo "Installing mdsd..." # fi # Install this way once moving to the Mariner published RPMs: -sudo tdnf install -y azure-mdsd-1.23.5 +sudo tdnf install -y azure-mdsd-1.27.4 cp -f $TMPDIR/envmdsd /etc/mdsd.d # Create the following directory for mdsd logs @@ -52,7 +52,7 @@ mkdir /opt/microsoft/linuxmonagent # Install telegraf echo "Installing telegraf..." -sudo tdnf install telegraf-1.25.2 -y +sudo tdnf install telegraf-1.27.3 -y sudo tdnf list installed | grep telegraf | awk '{print $2}' > telegrafversion.txt # Install fluent-bit @@ -64,7 +64,7 @@ cp /etc/cron.daily/logrotate /etc/cron.hourly/ # Install ME echo "Installing Metrics Extension..." -sudo tdnf install -y metricsext2-2.2023.224.2214 +sudo tdnf install -y metricsext2-2.2023.928.2134 sudo tdnf list installed | grep metricsext2 | awk '{print $2}' > metricsextversion.txt # tdnf does not have an autoremove feature. Only necessary packages are copied over to distroless build. Below reduces the image size if using non-distroless diff --git a/tools/az-prom-rules-converter/web-app-example/package-lock.json b/tools/az-prom-rules-converter/web-app-example/package-lock.json index 038d4c7c6..c0c8ea8d9 100644 --- a/tools/az-prom-rules-converter/web-app-example/package-lock.json +++ b/tools/az-prom-rules-converter/web-app-example/package-lock.json @@ -44,8 +44,9 @@ } }, "node_modules/@adobe/css-tools": { - "version": "4.0.1", - "license": "MIT" + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.3.1.tgz", + "integrity": "sha512-/62yikz7NLScCGAAST5SHdnjaDJQBDq0M2muyRTpf2VQhw6StBg2ALiu73zSJQ4fMVLA+0uBhBHAle7Wg+2kSg==" }, "node_modules/@ampproject/remapping": { "version": "2.2.0", @@ -15436,7 +15437,9 @@ }, "dependencies": { "@adobe/css-tools": { - "version": "4.0.1" + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.3.1.tgz", + "integrity": "sha512-/62yikz7NLScCGAAST5SHdnjaDJQBDq0M2muyRTpf2VQhw6StBg2ALiu73zSJQ4fMVLA+0uBhBHAle7Wg+2kSg==" }, "@ampproject/remapping": { "version": "2.2.0", From bbf5644f192409e96209f1f3ba783f94b8fd0a68 Mon Sep 17 00:00:00 2001 From: Vishwanath Date: Thu, 5 Oct 2023 21:58:26 -0700 Subject: [PATCH 3/5] remove private branch --- .pipelines/azure-pipeline-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/azure-pipeline-build.yml b/.pipelines/azure-pipeline-build.yml index 4bf1da9d4..f9e94cdcd 100644 --- a/.pipelines/azure-pipeline-build.yml +++ b/.pipelines/azure-pipeline-build.yml @@ -2,7 +2,6 @@ trigger: branches: include: - main - - vishwa/10022023-upgrade pr: autoCancel: true branches: From 8aef3451b18e190448da60cee84cb310a328b898 Mon Sep 17 00:00:00 2001 From: Vishwanath Date: Thu, 5 Oct 2023 23:40:05 -0700 Subject: [PATCH 4/5] Update RELEASENOTES.md --- RELEASENOTES.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index a90bad6e0..b1e24db47 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,29 @@ # Azure Monitor Metrics for AKS clusters +## Release 10-05-2023 +* Linux image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.7.7-main-` +* Windows image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.7.7-main-` +* Change log - + * Update k8s.io/clientgo in fluentbit plugin from `0.28.0` to `0.28.2` - https://github.com/Azure/prometheus-collector/pull/595 + * fix: ARC fixes (already released to ARC as hotfix) - https://github.com/Azure/prometheus-collector/pull/605 + * Update ARC regions (add Sweden South) + * Fix registry for node exporter + * Add `clusterDistribution` override parameter for AKS EE + * Update CPU requests for Daemonset (linux & windows) - https://github.com/Azure/prometheus-collector/pull/606 + * Add telemetry for per target scrape interval - https://github.com/Azure/prometheus-collector/pull/614 + * Update dependencies, Disable exemplars on ME (Linux only), Update CVE exemptions, Stop windows ingestion from replicaset, Fix try scans to fail on CVEs - https://github.com/Azure/prometheus-collector/pull/616 + * Linux + * mdsd = azure-mdsd-1.23.5 --> 1.27.4 + * ME = 2.2023.224.2214 --> 2.2023.928.2134 + * telegraf = 1.25.2 --> 1.27.3 + * golang = 1.18 --> 1.20 + * Windows + * golang = 1.18 --> 1.20 + * Upgrade addon token adapter for back door deployments (Linux only) + * master.221118.2 --> master.230804.1 + * Fix $ substitution issue in relabel and metric relabel config - https://github.com/Azure/prometheus-collector/pull/618 + * update github.com/prometheus/client_golang from `1.16.0` to `1.17.0` in fluentbit plugin - https://github.com/Azure/prometheus-collector/pull/608 + ## Release 9-11-2023 * Linux image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.7.5-main-09-12-2023-8fbde9ca` * Windows image - `mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-collector/images:6.7.5-main-09-12-2023-8fbde9ca-win` From 065e1594ff9e8b1e71fd6777ef2f756a99bdfcea Mon Sep 17 00:00:00 2001 From: Vishwanath Date: Thu, 5 Oct 2023 23:41:02 -0700 Subject: [PATCH 5/5] Update VERSION --- otelcollector/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otelcollector/VERSION b/otelcollector/VERSION index 2ed16f40d..38f118fae 100644 --- a/otelcollector/VERSION +++ b/otelcollector/VERSION @@ -1 +1 @@ -6.7.6 +6.7.7