feat(monitoring): add grafana + prometheus stack

FalcoSuessgott · Oct 30, 2023 · 573979b · 573979b
1 parent 9bbc0e5
commit 573979b
Show file tree

Hide file tree

Showing 22 changed files with 266 additions and 37 deletions.
diff --git a/docs/cm.md b/docs/cm.md
@@ -169,17 +169,17 @@ spec:
 An Ingress has been created, pointing to our Demo App and requesting a Certificate:
 
 ```bash
-$> cat k8s-cert-mananger/output/ingress.yml
+$> cat k8s-cert-mananger/output/kuard_ingress.yml
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   annotations:
-    cert-manager.io/issuer: vault-issuer
+    cert-manager.io/cluster-issuer: vault-issuer
   name: ingress
   namespace: cm
 spec:
   rules:
-  - host: 192.168.49.2.nip.io
+  - host: kuard.192.168.49.2.nip.io
     http:
       paths:
       - pathType: Prefix
@@ -191,8 +191,9 @@ spec:
               number: 80
   tls:
   - hosts:
-    - 192.168.49.2.nip.io
+    - kuard.192.168.49.2.nip.io
     secretName: kuard-cert
+
 ```
 
 The certificate was requested, signed and issued successfully and is stored as a kubernetes secret:
@@ -220,18 +221,32 @@ ca.crt:   1107 bytes
 tls.crt:  2392 bytes
 ```
 
+You can see the certificates in Vault:
+
+```bash
+# https://localhost/ui/vault/secrets/cert-manager-intermediate/pki/certificates
+$> vault list - cert-manager-intermediate/certs
+Keys
+----
+04:4b:a0:0d:b6:6e:45:04:1d:1a:30:4d:34:53:a6:e1:a3:31:59:8d
+0e:41:80:d7:36:71:e0:b7:8c:b8:88:54:3d:28:ab:0b:e8:cd:6a:59
+10:ec:5d:1a:9a:90:ac:af:14:57:85:05:35:cb:03:7d:ea:ce:ec:12
+5e:56:8e:52:f9:07:ea:d8:58:06:cb:bd:fa:af:8e:82:fc:82:95:82
+6f:48:4b:e8:29:5b:2a:4b:e9:cf:d5:68:5e:1a:c3:ce:81:9d:af:e0
+79:44:da:69:bf:21:e6:8d:27:fa:75:91:15:5f:11:20:19:23:8e:86
+```
 You can see that the connection to `kuard` is now secured and verified using the CA certificate:
 
 ```bash
 $> minikube profile vault-playground
-$> curl "https://$(minikube ip).nip.io"
+$> curl "https://kuard.$(minikube ip).nip.io"
 curl: (60) SSL certificate problem: unable to get local issuer certificate
 More details here: https://curl.se/docs/sslcerts.html
 
 curl failed to verify the legitimacy of the server and therefore could not
 establish a secure connection to it. To learn more about this situation and
 how to fix it, please visit the web page mentioned above.
-$> curl "https://$(minikube ip).nip.io" --cacert vault/ca.crt
+$> curl "https://kuard.$(minikube ip).nip.io" --cacert vault/ca.crt
 <!doctype html>
 ...
 ```
diff --git a/docs/d.md b/docs/d.md
diff --git a/docs/vai.md b/docs/vai.md
@@ -118,7 +118,7 @@ spec:
         vault.hashicorp.com/agent-inject-secret-secrets.txt: 'vai/data/secrets'
         vault.hashicorp.com/agent-inject-template-secrets.txt: |
           {{- with secret "vai/data/secrets" -}}
-          { 
+          {
             "username": "{{ .Data.data.username }}",
             "password": "{{ .Data.data.password }}"
           }

diff --git a/k8s-cert-manager/templates/dashboard_ingress.yml b/k8s-cert-manager/templates/dashboard_ingress.yml
@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    cert-manager.io/cluster-issuer: vault-issuer
+  name: dasboard
+  namespace: kubernetes-dashboard
+spec:
+  rules:
+  - host: dashboard.${minikube_ip}.nip.io
+    http:
+      paths:
+      - pathType: Prefix
+        path: /
+        backend:
+          service:
+            name: kubernetes-dashboard
+            port:
+              number: 80
+  tls:
+  - hosts:
+    - dashboard.${minikube_ip}.nip.io
+    secretName: dashboard-cert
diff --git a/k8s-cert-manager/templates/issuer.yml b/k8s-cert-manager/templates/issuer.yml
@@ -1,5 +1,5 @@
 apiVersion: cert-manager.io/v1
-kind: Issuer
+kind: ClusterIssuer
 metadata:
   name: vault-issuer
   namespace: cm

diff --git a/k8s-cert-manager/templates/ingress.yml → k8s-cert-manager/templates/kuard_ingress.yml b/k8s-cert-manager/templates/ingress.yml → k8s-cert-manager/templates/kuard_ingress.yml
@@ -2,12 +2,12 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   annotations:
-    cert-manager.io/issuer: vault-issuer
+    cert-manager.io/cluster-issuer: vault-issuer
   name: ingress
   namespace: cm
 spec:
   rules:
-  - host: ${minikube_ip}.nip.io
+  - host: kuard.${minikube_ip}.nip.io
     http:
       paths:
       - pathType: Prefix
@@ -19,5 +19,5 @@ spec:
               number: 80
   tls:
   - hosts:
-    - ${minikube_ip}.nip.io
+    - kuard.${minikube_ip}.nip.io
     secretName: kuard-cert
diff --git a/k8s-cert-manager/terraform/cm.tf b/k8s-cert-manager/terraform/cm.tf
@@ -61,16 +61,3 @@ resource "kubectl_manifest" "vault_issuer" {
 
   depends_on = [helm_release.cm]
 }
-
-resource "local_file" "ingress" {
-  filename = "${path.module}/../output/ingress.yml"
-  content = templatefile("${path.module}/../templates/ingress.yml", {
-    minikube_ip = var.minikube_ip
-  })
-}
-
-resource "kubectl_manifest" "ingress" {
-  yaml_body = local_file.ingress.content
-
-  depends_on = [helm_release.cm]
-}
diff --git a/k8s-cert-manager/terraform/dashboard.tf b/k8s-cert-manager/terraform/dashboard.tf
@@ -0,0 +1,13 @@
+
+resource "local_file" "dashboard" {
+  filename = "${path.module}/../output/dashboard_ingress.yml"
+  content = templatefile("${path.module}/../templates/dashboard_ingress.yml", {
+    minikube_ip = var.minikube_ip
+  })
+}
+
+resource "kubectl_manifest" "dashboard" {
+  yaml_body = local_file.dashboard.content
+
+  depends_on = [helm_release.cm]
+}
diff --git a/k8s-cert-manager/terraform/kuard.tf b/k8s-cert-manager/terraform/kuard.tf
@@ -9,3 +9,16 @@ resource "kubectl_manifest" "kuard_svc" {
 
   depends_on = [helm_release.cm]
 }
+
+resource "local_file" "ingress" {
+  filename = "${path.module}/../output/kuard_ingress.yml"
+  content = templatefile("${path.module}/../templates/kuard_ingress.yml", {
+    minikube_ip = var.minikube_ip
+  })
+}
+
+resource "kubectl_manifest" "ingress" {
+  yaml_body = local_file.ingress.content
+
+  depends_on = [helm_release.cm]
+}
diff --git a/k8s-external-secrets-operator/terraform/esm.tf b/k8s-external-secrets-operator/terraform/esm.tf
@@ -34,6 +34,9 @@ resource "local_file" "secret_store" {
 # Apply Secret Store CRD
 resource "kubectl_manifest" "secret_store" {
   yaml_body = local_file.secret_store.content
+
+  depends_on = [helm_release.esm]
+
 }
 
 # Render ExternalSecret CRD
@@ -52,4 +55,6 @@ resource "local_file" "external_secret" {
 # Apply CRD
 resource "kubectl_manifest" "external_secret" {
   yaml_body = local_file.external_secret.content
+
+  depends_on = [helm_release.esm]
 }
diff --git a/k8s-monitoring/files/scrape_config.yml b/k8s-monitoring/files/scrape_config.yml
@@ -0,0 +1,24 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ScrapeConfig
+metadata:
+  name: vault
+  namespace: monitoring
+spec:
+  metricsPath: v1/sys/metrics
+  params:
+    format: ['prometheus']
+  scheme: HTTPS
+  tlsConfig:
+    ca:
+      secret:
+        key: ca.crt
+        name: ca-cert
+  authorization:
+    credentials:
+      key: token
+      name: token
+  staticConfigs:
+    - labels:
+        job: prometheus
+      targets:
+        - https://host.minikube.internal
diff --git a/k8s-monitoring/files/vault-policy.hcl b/k8s-monitoring/files/vault-policy.hcl
@@ -0,0 +1,3 @@
+path "/sys/metrics" {
+  capabilities = ["read"]
+}
diff --git a/k8s-monitoring/files/vault_data_source.yml b/k8s-monitoring/files/vault_data_source.yml
@@ -0,0 +1,12 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: grafanadatasource-sample
+  namespace: monitoring
+spec:
+  datasource:
+    name: vault
+    type: prometheus
+    access: server
+    basicAuth: true
+    url: http://monitoring.kube-prometheus-prometheus.svc.prometheus-service:9090
diff --git a/k8s-monitoring/templates/values.yml b/k8s-monitoring/templates/values.yml
@@ -0,0 +1,50 @@
+# https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml
+grafana:
+  ingress:
+    enabled: true
+    ingressClassName: nginx
+    annotations:
+      cert-manager.io/cluster-issuer: vault-issuer
+    hosts:
+      - grafana.${minikube_ip}.nip.io
+    paths:
+      - /
+    tls:
+      - secretName: grafana-cert
+        hosts:
+          - grafana.${minikube_ip}.nip.io
+  # dashboards:
+  #   default:
+  #     nginx-ingress:
+  #       url: https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/grafana/dashboards/nginx.json
+
+prometheus:
+  ingress:
+    enabled: true
+    ingressClassName: nginx
+    annotations:
+      cert-manager.io/cluster-issuer: vault-issuer
+    hosts:
+      - prometheus.${minikube_ip}.nip.io
+    paths:
+      - /
+    tls:
+      - secretName: prometheus-cert
+        hosts:
+          - prometheus.${minikube_ip}.nip.io
+
+# Disable Etcd metrics
+kubeEtcd:
+  enabled: false
+
+# Disable Controller metrics
+kubeControllerManager:
+  enabled: false
+
+# Disable Scheduler metrics
+kubeScheduler:
+  enabled: false
+
+# Disable alertmanager
+alertmanager:
+  enabled: false
diff --git a/k8s-monitoring/terraform/main.tf b/k8s-monitoring/terraform/main.tf
@@ -0,0 +1,26 @@
+# Install VAI via Helm
+resource "helm_release" "prometheus" {
+  name             = "monitoring"
+  repository       = "https://prometheus-community.github.io/helm-charts"
+  chart            = "kube-prometheus-stack"
+  namespace        = "monitoring"
+  create_namespace = true
+
+  values = [templatefile(
+    "${path.module}/../templates/values.yml",
+    {
+      minikube_ip = var.minikube_ip
+    }
+  )]
+}
+
+resource "kubernetes_secret" "ca_cert" {
+  metadata {
+    name      = "ca-cert"
+    namespace = helm_release.prometheus.namespace
+  }
+
+  data = {
+    "ca.crt" = var.ca_cert
+  }
+}
diff --git a/k8s-monitoring/terraform/variables.tf b/k8s-monitoring/terraform/variables.tf
@@ -0,0 +1,7 @@
+variable "ca_cert" {
+  type = string
+}
+
+variable "minikube_ip" {
+  type = string
+}
diff --git a/k8s-monitoring/terraform/vault.tf b/k8s-monitoring/terraform/vault.tf
@@ -0,0 +1,25 @@
+resource "vault_policy" "prometheus" {
+  name = "prometheus"
+
+  policy = file("${path.module}/../files/vault-policy.hcl")
+}
+
+
+resource "vault_token" "prometheus" {
+  policies = [vault_policy.prometheus.name]
+
+  renewable = true
+  period    = "24h"
+  no_parent = true
+}
+
+resource "kubernetes_secret" "token" {
+  metadata {
+    name      = "token"
+    namespace = helm_release.prometheus.namespace
+  }
+
+  data = {
+    "token" = vault_token.prometheus.client_token
+  }
+}
diff --git a/k8s-monitoring/terraform/version.tf b/k8s-monitoring/terraform/version.tf
@@ -0,0 +1,26 @@
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    vault = {
+      source  = "hashicorp/vault"
+      version = "3.20.1"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = "2.4.0"
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.23.0"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.11.0"
+    }
+    kubectl = {
+      source  = "gavinbunney/kubectl"
+      version = "1.14.0"
+    }
+  }
+}
diff --git a/k8s-vault-agent-injector/terraform/kuard.tf b/k8s-vault-agent-injector/terraform/kuard.tf
@@ -2,4 +2,4 @@ resource "kubectl_manifest" "kuard" {
   yaml_body = file("${path.module}/../files/kuard.yml")
 
   depends_on = [helm_release.vai]
-}
+}
diff --git a/main.tf b/main.tf
@@ -94,3 +94,12 @@ module "cm" {
 
   depends_on = [module.vault_k8s]
 }
+
+module "monitoring" {
+  source = "./k8s-monitoring/terraform"
+
+  ca_cert     = module.tls.ca.cert
+  minikube_ip = module.minikube[0].minikube_ip
+
+  depends_on = [module.vault_k8s]
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,4 +2,4 @@ resource "kubectl_manifest" "kuard" { @@
       yaml_body = file("${path.module}/../files/kuard.yml")
       depends_on = [helm_release.vai]
-    }
+    }