Add monitoring to OKD

Dockerfile.ubuntu

@@ -49,6 +49,7 @@ RUN apt update && \
       curl \
       dialog \
       dnsutils \
+      gawk \
       gettext \
       git \
       iproute2 \

templates/manifests/base/helmrelease-x509-exporter.yaml.tpl

@@ -3,8 +3,11 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: x509-exporter
+%{~ if cluster_type == "okd" }
   labels:
-    openshift.io/cluster-monitoring: "true"
+    openshift.io/cluster-monitoring: "false"
+    openshift.io/user-monitoring: "true"
+%{~ endif }
 ---
 %{~ if cluster_type == "okd" }
 apiVersion: rbac.authorization.k8s.io/v1
@@ -136,17 +139,21 @@ spec:
             operator: Exists
 
 %{~ if cluster_type == "okd" }
-          watchDirectories:
-          - /var/lib/kubelet/pki
+          watchFiles:
+          - /var/lib/kubelet/pki/kubelet-server-current.pem
+          - /var/lib/kubelet/pki/kubelet-client-current.pem
 
           watchKubeconfFiles:
           - /etc/kubernetes/kubeconfig
           - /etc/kubernetes/kubelet.conf
 %{~ endif }
 %{~ if cluster_type == "kubespray" }
+          watchFiles:
+          - /var/lib/kubelet/pki/kubelet-server-current.pem
+          - /var/lib/kubelet/pki/kubelet-client-current.pem
+
           watchDirectories:
           - /etc/kubernetes/ssl
-          - /var/lib/kubelet/pki
 
           watchKubeconfFiles:
           - /etc/kubernetes/kubelet.conf
@@ -169,6 +176,14 @@ spec:
     #prometheusRules:
     #  create: false
 
+    prometheusServiceMonitor:
+      create: true
+      scrapeInterval: 600s
+    prometheusRules:
+      create: true
+      warningDaysLeft: 14
+      criticalDaysLeft: 7
+
     rbac:
       secretsExporter:
         serviceAccountName: x509-exporter-secrets

templates/okd/main.tf

@@ -7,7 +7,7 @@ module "cluster" {
   cluster_sla      = var.cluster_sla
   cluster_type     = local.cluster_type
   cluster_provider = var.cluster_provider
-  use_kubeconfig   = var.use_kubeconfig
+  use_kubeconfig   = true
   pre_create       = var.pre_create
   post_create      = var.post_create
   modules          = local.modules_result

templates/okd/manifests/cluster/kustomization.yaml

@@ -15,30 +15,30 @@ patchesStrategicMerge: []
 #- helmrelease-logging.yaml
 #- excludes.yaml
 
-patches:
-- target:
-    kind: Deployment
-    name: efs-csi-controller
-    namespace: kube-system
-  patch: |-
-    - op: add
-      path: /spec/template/spec/hostNetwork
-      value: true
-    - op: replace
-      path: /spec/template/spec/containers/0/args/3
-      value: "--delete-access-point-root-dir=true"
-    - op: add
-      path: /spec/template/spec/containers/0/env/-
-      value: 
-        name: AWS_USE_FIPS_ENDPOINT
-        value: "true"
-- target:
-    kind: DaemonSet
-    name: efs-csi-node
-    namespace: kube-system
-  patch: |-
-    - op: add
-      path: /spec/template/spec/containers/0/env/-
-      value: 
-        name: AWS_USE_FIPS_ENDPOINT
-        value: "true"
+patches: []
+#- target:
+#    kind: Deployment
+#    name: efs-csi-controller
+#    namespace: kube-system
+#  patch: |-
+#    - op: add
+#      path: /spec/template/spec/hostNetwork
+#      value: true
+#    - op: replace
+#      path: /spec/template/spec/containers/0/args/3
+#      value: "--delete-access-point-root-dir=true"
+#    - op: add
+#      path: /spec/template/spec/containers/0/env/-
+#      value:
+#        name: AWS_USE_FIPS_ENDPOINT
+#        value: "true"
+#- target:
+#    kind: DaemonSet
+#    name: efs-csi-node
+#    namespace: kube-system
+#  patch: |-
+#    - op: add
+#      path: /spec/template/spec/containers/0/env/-
+#      value:
+#        name: AWS_USE_FIPS_ENDPOINT
+#        value: "true"

templates/okd/manifests/provider/helmrelease-linkerd.yaml

@@ -0,0 +1,7 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: linkerd-viz
+  namespace: linkerd-viz
+spec:
+  ingressClassName: openshift-default

templates/okd/manifests/provider/kustomization.yaml

@@ -2,7 +2,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ../base/
+# Do not include all from ../base in order to use openshift native monitoring system
+- ../base/helmrepository.yaml
+- ../base/helmrelease-teleport-agent.yaml
+- ../base/helmrelease-x509-exporter.yaml
 - helmrelease-cert-manager.yaml
 - helmrelease-cert-manager-config.yaml
 - helmrelease-cert-utils-operator.yaml

templates/okd/monitoring.tf

@@ -0,0 +1,42 @@
+resource "kubernetes_config_map_v1" "cluster-monitoring-config" {
+  metadata {
+    name      = "cluster-monitoring-config"
+    namespace = "openshift-monitoring"
+  }
+
+  data = {
+    "config.yaml" = <<-EOT
+      enableUserWorkload: true
+      prometheusK8s:
+        resources:
+          requests:
+            memory: 4Gi
+            cpu: 1
+          limits:
+            memory: 12Gi
+            cpu: 2
+          retention: 30d
+          retentionSize: 50GiB
+        logLevel: info
+    EOT
+  }
+}
+
+resource "kubernetes_config_map_v1" "user-workload-monitoring-config" {
+  metadata {
+    name      = "user-workload-monitoring-config"
+    namespace = "openshift-user-workload-monitoring"
+  }
+
+  data = {
+    "config.yaml" = <<-EOT
+      prometheus:
+        retention: 24h
+        retentionSize: 10GiB
+      alertmanager:
+        enabled: true
+        enableAlertmanagerConfig: true
+        logLevel: info
+    EOT
+  }
+}

templates/okd/providers-okd.tf

@@ -0,0 +1,9 @@
+provider "kubernetes" {
+  config_path = local.kubeconfig_filename
+}
+
+provider "kubectl" {
+  load_config_file  = true
+  config_path       = local.kubeconfig_filename
+  apply_retry_count = 2
+}