From 6fd3ab163a00bf0c53ac94cc18bfad41f12ed1a3 Mon Sep 17 00:00:00 2001 From: Chris Werner Rau Date: Thu, 22 Feb 2024 14:08:05 +0100 Subject: [PATCH] feat(t8s-cluster/management-cluster): automatically roll cluster if certs are going to expire (#722) --- .../_helpers.tpl | 0 .../_kubeadmControlPlaneTemplateSpec.yaml | 77 +++++++++++++++++++ .../kubeadmControlPlaneTemplate.yaml | 3 +- .../_kubeadmControlPlaneTemplateSpec.yaml | 74 ------------------ 4 files changed, 78 insertions(+), 76 deletions(-) rename charts/t8s-cluster/templates/management-cluster/clusterClass/{kubeadmnControlPlaneTemplate => kubeadmControlPlaneTemplate}/_helpers.tpl (100%) create mode 100644 charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml rename charts/t8s-cluster/templates/management-cluster/clusterClass/{kubeadmnControlPlaneTemplate => kubeadmControlPlaneTemplate}/kubeadmControlPlaneTemplate.yaml (68%) delete mode 100644 charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl similarity index 100% rename from charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_helpers.tpl rename to charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml new file mode 100644 index 000000000..f1b792d3d --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml @@ -0,0 +1,77 @@ +{{/* +KubeadmControlPlaneTemplate is immutable. We need to create new versions during upgrades. +Here we are generating a hash suffix. +This function needs the whole `$` context to be able to use `.Files.Get` +*/}} +{{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" -}} +rolloutBefore: + certificatesExpiryDays: 60 +kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + admission-control-config-file: &admissionControlConfigFilePath /etc/kubernetes/admission-control-config.yaml + cloud-provider: external + enable-admission-plugins: AlwaysPullImages,EventRateLimit,NodeRestriction + profiling: 'false' + tls-cipher-suites: {{ include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) }} + event-ttl: 4h + extraVolumes: + - hostPath: *admissionControlConfigFilePath + mountPath: *admissionControlConfigFilePath + name: admission-control-config + readOnly: true + - hostPath: &eventRateLimitConfigFilePath /etc/kubernetes/event-rate-limit-config.yaml + mountPath: *eventRateLimitConfigFilePath + name: event-rate-limit-config + readOnly: true + controllerManager: + extraArgs: + authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics + bind-address: 0.0.0.0 + cloud-provider: external + profiling: 'false' + terminated-pod-gc-threshold: '100' + etcd: + local: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2381 + scheduler: + extraArgs: + authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics + bind-address: 0.0.0.0 + profiling: 'false' + files: {{- include "t8s-cluster.patches.kubelet.patches" (dict "context" $) | nindent 4 }} + - content: |- {{- .Files.Get "files/admission-control-config.yaml" | nindent 8 }} + path: *admissionControlConfigFilePath + - content: |- {{- .Files.Get "files/event-rate-limit-config.yaml" | nindent 8 }} + path: *eventRateLimitConfigFilePath + - content: |- {{- .Files.Get "files/kube-proxy.patch.sh" | nindent 8 }} + path: /etc/kube-proxy-patch.sh + permissions: "0700" + - content: |- {{- .Files.Get "files/kube-proxy.config.yaml" | nindent 8 }} + path: /etc/kube-proxy-config.yaml + {{- if .Values.containerRegistryMirror.mirrorEndpoint }} + {{- include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" $) | nindent 4 }} + {{- end }} + - content: |- {{- include "t8s-cluster.clusterClass.containerdConfig.plugins" (dict "context" $ "gpu" false) | nindent 8 }} + path: /etc/containerd/conf.d/plugins.toml + {{- if .Values.global.injectedCertificateAuthorities }} + - content: |- {{- .Values.global.injectedCertificateAuthorities | nindent 8 }} + path: /usr/local/share/ca-certificates/injected-ca-certs.crt + {{- end }} + initConfiguration: + nodeRegistration: + kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 8 }} + name: '{{ `{{ local_hostname }}` }}' + patches: + directory: {{ include "t8s-cluster.patches.directory" (dict) }} + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 8 }} + name: '{{ `{{ local_hostname }}` }}' + patches: + directory: {{ include "t8s-cluster.patches.directory" (dict) }} + preKubeadmCommands: {{- include "t8s-cluster.clusterClass.preKubeadmCommands" (dict "context" $) | nindent 4 }} + postKubeadmCommands: {{- include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | nindent 4 }} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml similarity index 68% rename from charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml rename to charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml index 3ef2fd51c..1022d7498 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml @@ -6,5 +6,4 @@ metadata: labels: {{- include "common.labels.standard" $ | nindent 4 }} spec: template: - spec: - kubeadmConfigSpec: {{/* the full context is needed for .Files.Get */}}{{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" . | nindent 8 }} + spec: {{/* the full context is needed for .Files.Get */}}{{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" . | nindent 6 }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml deleted file mode 100644 index 332ad9e1c..000000000 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml +++ /dev/null @@ -1,74 +0,0 @@ -{{/* -KubeadmControlPlaneTemplate is immutable. We need to create new versions during upgrades. -Here we are generating a hash suffix. -This function needs the whole `$` context to be able to use `.Files.Get` -*/}} -{{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" -}} -clusterConfiguration: - apiServer: - extraArgs: - admission-control-config-file: &admissionControlConfigFilePath /etc/kubernetes/admission-control-config.yaml - cloud-provider: external - enable-admission-plugins: AlwaysPullImages,EventRateLimit,NodeRestriction - profiling: 'false' - tls-cipher-suites: {{ include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) }} - event-ttl: 4h - extraVolumes: - - hostPath: *admissionControlConfigFilePath - mountPath: *admissionControlConfigFilePath - name: admission-control-config - readOnly: true - - hostPath: &eventRateLimitConfigFilePath /etc/kubernetes/event-rate-limit-config.yaml - mountPath: *eventRateLimitConfigFilePath - name: event-rate-limit-config - readOnly: true - controllerManager: - extraArgs: - authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics - bind-address: 0.0.0.0 - cloud-provider: external - profiling: 'false' - terminated-pod-gc-threshold: '100' - etcd: - local: - extraArgs: - listen-metrics-urls: http://0.0.0.0:2381 - scheduler: - extraArgs: - authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics - bind-address: 0.0.0.0 - profiling: 'false' -files: {{- include "t8s-cluster.patches.kubelet.patches" (dict "context" $) | nindent 2 }} - - content: |- {{- .Files.Get "files/admission-control-config.yaml" | nindent 6 }} - path: *admissionControlConfigFilePath - - content: |- {{- .Files.Get "files/event-rate-limit-config.yaml" | nindent 6 }} - path: *eventRateLimitConfigFilePath - - content: |- {{- .Files.Get "files/kube-proxy.patch.sh" | nindent 6 }} - path: /etc/kube-proxy-patch.sh - permissions: "0700" - - content: |- {{- .Files.Get "files/kube-proxy.config.yaml" | nindent 6 }} - path: /etc/kube-proxy-config.yaml - {{- if .Values.containerRegistryMirror.mirrorEndpoint }} - {{- include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" $) | nindent 2 }} - {{- end }} - - content: |- {{- include "t8s-cluster.clusterClass.containerdConfig.plugins" (dict "context" $ "gpu" false) | nindent 6 }} - path: /etc/containerd/conf.d/plugins.toml - {{- if .Values.global.injectedCertificateAuthorities }} - - content: |- {{- .Values.global.injectedCertificateAuthorities | nindent 6 }} - path: /usr/local/share/ca-certificates/injected-ca-certs.crt - {{- end }} -initConfiguration: - nodeRegistration: - kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 6 }} - name: '{{ `{{ local_hostname }}` }}' - patches: - directory: {{ include "t8s-cluster.patches.directory" (dict) }} -joinConfiguration: - nodeRegistration: - kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 6 }} - name: '{{ `{{ local_hostname }}` }}' - patches: - directory: {{ include "t8s-cluster.patches.directory" (dict) }} -preKubeadmCommands: {{- include "t8s-cluster.clusterClass.preKubeadmCommands" (dict "context" $) | nindent 2 }} -postKubeadmCommands: {{- include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | nindent 2 }} -{{- end -}}