From 3d25f184d04035aab134afeee4eac1779ecc28c9 Mon Sep 17 00:00:00 2001 From: Ahree Hong Date: Thu, 30 May 2024 08:41:34 -0700 Subject: [PATCH] Add control plane tolerations to deployments Signed-off-by: Ahree Hong --- tinkerbell/hegel/Chart.yaml | 2 +- tinkerbell/hegel/templates/_scheduling.tpl | 12 ++++++++++++ tinkerbell/hegel/templates/deployment.yaml | 8 ++++++++ tinkerbell/hegel/values.yaml | 5 +++++ tinkerbell/rufio/Chart.yaml | 2 +- tinkerbell/rufio/templates/_scheduling.tpl | 12 ++++++++++++ tinkerbell/rufio/templates/deployment.yaml | 8 ++++++++ tinkerbell/rufio/values.yaml | 6 +++++- tinkerbell/smee/Chart.yaml | 2 +- tinkerbell/smee/templates/_scheduling.tpl | 12 ++++++++++++ tinkerbell/smee/templates/deployment.yaml | 8 ++++++++ tinkerbell/smee/values.yaml | 5 +++++ tinkerbell/stack/Chart.lock | 12 ++++++------ tinkerbell/stack/Chart.yaml | 8 ++++---- tinkerbell/stack/templates/_scheduling.tpl | 12 ++++++++++++ tinkerbell/stack/templates/kubevip.yaml | 4 ++++ tinkerbell/stack/templates/nginx.yaml | 8 ++++++++ tinkerbell/stack/values.yaml | 4 ++++ tinkerbell/tink/Chart.yaml | 2 +- .../tink/templates/tink-controller/_scheduling.tpl | 12 ++++++++++++ .../tink/templates/tink-controller/deployment.yaml | 8 ++++++++ .../tink/templates/tink-server/_scheduling.tpl | 12 ++++++++++++ .../tink/templates/tink-server/deployment.yaml | 8 ++++++++ tinkerbell/tink/values.yaml | 8 ++++++++ 24 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 tinkerbell/hegel/templates/_scheduling.tpl create mode 100644 tinkerbell/rufio/templates/_scheduling.tpl create mode 100644 tinkerbell/smee/templates/_scheduling.tpl create mode 100644 tinkerbell/stack/templates/_scheduling.tpl create mode 100644 tinkerbell/tink/templates/tink-controller/_scheduling.tpl create mode 100644 tinkerbell/tink/templates/tink-server/_scheduling.tpl diff --git a/tinkerbell/hegel/Chart.yaml b/tinkerbell/hegel/Chart.yaml index 2bdd526c..8c3d0dbd 100644 --- a/tinkerbell/hegel/Chart.yaml +++ b/tinkerbell/hegel/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.5 +version: 0.3.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/tinkerbell/hegel/templates/_scheduling.tpl b/tinkerbell/hegel/templates/_scheduling.tpl new file mode 100644 index 00000000..395860de --- /dev/null +++ b/tinkerbell/hegel/templates/_scheduling.tpl @@ -0,0 +1,12 @@ +{{- define "singleNodeClusterConfig" }} +- effect: NoSchedule + key: node-role.kubernetes.io/control-plane +{{- end }} + +{{- define "preferWorkerNodes" }} +- weight: {{ .nodeAffinityWeight }} + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist +{{- end }} diff --git a/tinkerbell/hegel/templates/deployment.yaml b/tinkerbell/hegel/templates/deployment.yaml index 25b33fed..519275d2 100644 --- a/tinkerbell/hegel/templates/deployment.yaml +++ b/tinkerbell/hegel/templates/deployment.yaml @@ -56,4 +56,12 @@ spec: cpu: {{ .Values.resources.requests.cpu }} memory: {{ .Values.resources.requests.memory }} serviceAccountName: {{ .Values.name }} + {{- if .Values.singleNodeClusterConfig.controlPlaneTolerationsEnabled }} + tolerations: + {{- include "singleNodeClusterConfig" . | indent 6 }} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "preferWorkerNodes" (dict "nodeAffinityWeight" .Values.singleNodeClusterConfig.nodeAffinityWeight) | indent 10 }} + {{- end }} {{- end }} diff --git a/tinkerbell/hegel/values.yaml b/tinkerbell/hegel/values.yaml index 97b47f29..b2f88273 100644 --- a/tinkerbell/hegel/values.yaml +++ b/tinkerbell/hegel/values.yaml @@ -22,3 +22,8 @@ nodeSelector: {} # Trusted proxies defines a list of IP or CIDR ranges that are allowed to set the X-Forwarded-For # header. This typically requires all Pod CIDRs in the cluster. trustedProxies: [] + +# singleNodeClusterConfig to add tolerations for deployments on control plane nodes. This is defaulted to false. +singleNodeClusterConfig: + controlPlaneTolerationsEnabled: false + nodeAffinityWeight: 1 diff --git a/tinkerbell/rufio/Chart.yaml b/tinkerbell/rufio/Chart.yaml index cba449e5..f0f63c1d 100644 --- a/tinkerbell/rufio/Chart.yaml +++ b/tinkerbell/rufio/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.9 +version: 0.2.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/tinkerbell/rufio/templates/_scheduling.tpl b/tinkerbell/rufio/templates/_scheduling.tpl new file mode 100644 index 00000000..395860de --- /dev/null +++ b/tinkerbell/rufio/templates/_scheduling.tpl @@ -0,0 +1,12 @@ +{{- define "singleNodeClusterConfig" }} +- effect: NoSchedule + key: node-role.kubernetes.io/control-plane +{{- end }} + +{{- define "preferWorkerNodes" }} +- weight: {{ .nodeAffinityWeight }} + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist +{{- end }} diff --git a/tinkerbell/rufio/templates/deployment.yaml b/tinkerbell/rufio/templates/deployment.yaml index 962db403..279eccf9 100644 --- a/tinkerbell/rufio/templates/deployment.yaml +++ b/tinkerbell/rufio/templates/deployment.yaml @@ -63,4 +63,12 @@ spec: memory: {{ .Values.resources.requests.memory }} serviceAccountName: {{ .Values.serviceAccountName }} terminationGracePeriodSeconds: 10 + {{- if .Values.singleNodeClusterConfig.controlPlaneTolerationsEnabled }} + tolerations: + {{- include "singleNodeClusterConfig" . | indent 6 }} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "preferWorkerNodes" (dict "nodeAffinityWeight" .Values.singleNodeClusterConfig.nodeAffinityWeight) | indent 10 }} + {{- end }} {{- end }} diff --git a/tinkerbell/rufio/values.yaml b/tinkerbell/rufio/values.yaml index 39589bbe..174b651e 100644 --- a/tinkerbell/rufio/values.yaml +++ b/tinkerbell/rufio/values.yaml @@ -15,4 +15,8 @@ rufioLeaderElectionRoleName: rufio-leader-election-role managerRoleName: rufio-manager-role rufioLeaderElectionRoleBindingName: rufio-leader-election-rolebinding managerRoleBindingName: rufio-manager-rolebinding -nodeSelector: {} \ No newline at end of file +nodeSelector: {} +# singleNodeClusterConfig to add tolerations for deployments on control plane nodes. This is defaulted to false. +singleNodeClusterConfig: + controlPlaneTolerationsEnabled: false + nodeAffinityWeight: 1 diff --git a/tinkerbell/smee/Chart.yaml b/tinkerbell/smee/Chart.yaml index 14b30f78..e3e9ed0d 100644 --- a/tinkerbell/smee/Chart.yaml +++ b/tinkerbell/smee/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.4 +version: 0.3.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/tinkerbell/smee/templates/_scheduling.tpl b/tinkerbell/smee/templates/_scheduling.tpl new file mode 100644 index 00000000..395860de --- /dev/null +++ b/tinkerbell/smee/templates/_scheduling.tpl @@ -0,0 +1,12 @@ +{{- define "singleNodeClusterConfig" }} +- effect: NoSchedule + key: node-role.kubernetes.io/control-plane +{{- end }} + +{{- define "preferWorkerNodes" }} +- weight: {{ .nodeAffinityWeight }} + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist +{{- end }} diff --git a/tinkerbell/smee/templates/deployment.yaml b/tinkerbell/smee/templates/deployment.yaml index e0db8746..e7c0af54 100644 --- a/tinkerbell/smee/templates/deployment.yaml +++ b/tinkerbell/smee/templates/deployment.yaml @@ -88,4 +88,12 @@ spec: {{- if .Values.hostNetwork }} hostNetwork: true {{- end }} + {{- if .Values.singleNodeClusterConfig.controlPlaneTolerationsEnabled }} + tolerations: + {{- include "singleNodeClusterConfig" . | indent 6 }} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "preferWorkerNodes" (dict "nodeAffinityWeight" .Values.singleNodeClusterConfig.nodeAffinityWeight) | indent 10 }} + {{- end }} {{- end }} diff --git a/tinkerbell/smee/values.yaml b/tinkerbell/smee/values.yaml index c7f5d5f9..3dd015e4 100644 --- a/tinkerbell/smee/values.yaml +++ b/tinkerbell/smee/values.yaml @@ -129,3 +129,8 @@ additionalArgs: [] # - name: MY_ENV_VAR # value: my-value additionalEnv: [] + +# singleNodeClusterConfig to add tolerations for deployments on control plane nodes. This is defaulted to false. +singleNodeClusterConfig: + controlPlaneTolerationsEnabled: false + nodeAffinityWeight: 1 diff --git a/tinkerbell/stack/Chart.lock b/tinkerbell/stack/Chart.lock index 3bcc99d0..d7e60312 100644 --- a/tinkerbell/stack/Chart.lock +++ b/tinkerbell/stack/Chart.lock @@ -1,15 +1,15 @@ dependencies: - name: tink repository: file://../tink - version: 0.2.4 + version: 0.2.5 - name: smee repository: file://../smee - version: 0.3.4 + version: 0.3.5 - name: rufio repository: file://../rufio - version: 0.2.9 + version: 0.2.10 - name: hegel repository: file://../hegel - version: 0.3.5 -digest: sha256:8adf6b3a5dea3c6a5ccb1cd5a32619a6fa9f79d0ac23d8601b95687dd63aac59 -generated: "2024-06-04T15:48:10.744939275-06:00" + version: 0.3.6 +digest: sha256:5ddb03f83b97f945fafaff989c86da72691f388da0edc33cd85782392273c21b +generated: "2024-06-10T16:59:58.906762-07:00" diff --git a/tinkerbell/stack/Chart.yaml b/tinkerbell/stack/Chart.yaml index 696b135d..6610ede9 100644 --- a/tinkerbell/stack/Chart.yaml +++ b/tinkerbell/stack/Chart.yaml @@ -25,14 +25,14 @@ appVersion: "0.4.5" dependencies: - name: tink - version: "0.2.4" + version: "0.2.5" repository: "file://../tink" - name: smee - version: "0.3.4" + version: "0.3.5" repository: "file://../smee" - name: rufio - version: "0.2.9" + version: "0.2.10" repository: "file://../rufio" - name: hegel - version: "0.3.5" + version: "0.3.6" repository: "file://../hegel" diff --git a/tinkerbell/stack/templates/_scheduling.tpl b/tinkerbell/stack/templates/_scheduling.tpl new file mode 100644 index 00000000..395860de --- /dev/null +++ b/tinkerbell/stack/templates/_scheduling.tpl @@ -0,0 +1,12 @@ +{{- define "singleNodeClusterConfig" }} +- effect: NoSchedule + key: node-role.kubernetes.io/control-plane +{{- end }} + +{{- define "preferWorkerNodes" }} +- weight: {{ .nodeAffinityWeight }} + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist +{{- end }} diff --git a/tinkerbell/stack/templates/kubevip.yaml b/tinkerbell/stack/templates/kubevip.yaml index 951ba1b4..f794fa1f 100644 --- a/tinkerbell/stack/templates/kubevip.yaml +++ b/tinkerbell/stack/templates/kubevip.yaml @@ -45,6 +45,10 @@ spec: - NET_RAW hostNetwork: true serviceAccountName: {{ .Values.stack.kubevip.name }} + {{- if .Values.stack.singleNodeClusterConfig.controlPlaneTolerationsEnabled }} + tolerations: + {{- include "singleNodeClusterConfig" (dict "nodeAffinityWeight" .Values.stack.singleNodeClusterConfig.nodeAffinityWeight) | indent 6 }} + {{- end }} --- apiVersion: v1 kind: ServiceAccount diff --git a/tinkerbell/stack/templates/nginx.yaml b/tinkerbell/stack/templates/nginx.yaml index b31f89dd..abc5bb6a 100644 --- a/tinkerbell/stack/templates/nginx.yaml +++ b/tinkerbell/stack/templates/nginx.yaml @@ -160,6 +160,14 @@ spec: securityContext: privileged: true {{- end }} + {{- if .Values.stack.singleNodeClusterConfig.controlPlaneTolerationsEnabled }} + tolerations: + {{- include "singleNodeClusterConfig" . | indent 6 }} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "preferWorkerNodes" (dict "nodeAffinityWeight" .Values.stack.singleNodeClusterConfig.nodeAffinityWeight) | indent 10 }} + {{- end }} {{- if .Values.stack.service.enabled }} --- apiVersion: v1 diff --git a/tinkerbell/stack/values.yaml b/tinkerbell/stack/values.yaml index 52f8fded..57f22445 100644 --- a/tinkerbell/stack/values.yaml +++ b/tinkerbell/stack/values.yaml @@ -67,6 +67,10 @@ stack: # interfaceMode determines how we create the interface needed to listen for DHCP broadcast traffic. # by default macvlan is used. ipvlan is the only other option. interfaceMode: macvlan + # singleNodeClusterConfig to add tolerations for deployments on control plane nodes. This is defaulted to false. + singleNodeClusterConfig: + controlPlaneTolerationsEnabled: false + nodeAffinityWeight: 1 # -- Overrides # The values defined here override those in the individual charts. Some of them require tweaking diff --git a/tinkerbell/tink/Chart.yaml b/tinkerbell/tink/Chart.yaml index d9906bc8..e5691925 100644 --- a/tinkerbell/tink/Chart.yaml +++ b/tinkerbell/tink/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.4 +version: 0.2.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/tinkerbell/tink/templates/tink-controller/_scheduling.tpl b/tinkerbell/tink/templates/tink-controller/_scheduling.tpl new file mode 100644 index 00000000..395860de --- /dev/null +++ b/tinkerbell/tink/templates/tink-controller/_scheduling.tpl @@ -0,0 +1,12 @@ +{{- define "singleNodeClusterConfig" }} +- effect: NoSchedule + key: node-role.kubernetes.io/control-plane +{{- end }} + +{{- define "preferWorkerNodes" }} +- weight: {{ .nodeAffinityWeight }} + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist +{{- end }} diff --git a/tinkerbell/tink/templates/tink-controller/deployment.yaml b/tinkerbell/tink/templates/tink-controller/deployment.yaml index 30a35bba..13d6723b 100644 --- a/tinkerbell/tink/templates/tink-controller/deployment.yaml +++ b/tinkerbell/tink/templates/tink-controller/deployment.yaml @@ -38,4 +38,12 @@ spec: cpu: {{ .Values.controller.resources.requests.cpu }} memory: {{ .Values.controller.resources.requests.memory }} serviceAccountName: {{ .Values.controller.name }} + {{- if .Values.controller.singleNodeClusterConfig.controlPlaneTolerationsEnabled }} + tolerations: + {{- include "singleNodeClusterConfig" . | indent 6 }} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "preferWorkerNodes" (dict "nodeAffinityWeight" .Values.controller.singleNodeClusterConfig.nodeAffinityWeight) | indent 10 }} + {{- end }} {{- end }} diff --git a/tinkerbell/tink/templates/tink-server/_scheduling.tpl b/tinkerbell/tink/templates/tink-server/_scheduling.tpl new file mode 100644 index 00000000..395860de --- /dev/null +++ b/tinkerbell/tink/templates/tink-server/_scheduling.tpl @@ -0,0 +1,12 @@ +{{- define "singleNodeClusterConfig" }} +- effect: NoSchedule + key: node-role.kubernetes.io/control-plane +{{- end }} + +{{- define "preferWorkerNodes" }} +- weight: {{ .nodeAffinityWeight }} + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist +{{- end }} diff --git a/tinkerbell/tink/templates/tink-server/deployment.yaml b/tinkerbell/tink/templates/tink-server/deployment.yaml index 7e16e790..ef380263 100644 --- a/tinkerbell/tink/templates/tink-server/deployment.yaml +++ b/tinkerbell/tink/templates/tink-server/deployment.yaml @@ -48,4 +48,12 @@ spec: cpu: {{ .Values.server.resources.requests.cpu }} memory: {{ .Values.server.resources.requests.memory }} serviceAccountName: {{ .Values.server.name }} + {{- if .Values.server.singleNodeClusterConfig.controlPlaneTolerationsEnabled }} + tolerations: + {{- include "singleNodeClusterConfig" . | indent 6 }} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "preferWorkerNodes" (dict "nodeAffinityWeight" .Values.server.singleNodeClusterConfig.nodeAffinityWeight) | indent 10 }} + {{- end }} {{- end }} diff --git a/tinkerbell/tink/values.yaml b/tinkerbell/tink/values.yaml index dbd5ba56..bd6a2c76 100644 --- a/tinkerbell/tink/values.yaml +++ b/tinkerbell/tink/values.yaml @@ -17,6 +17,10 @@ controller: tinkLeaderElectionRoleName: tink-leader-election-role tinkLeaderElectionRoleBindingName: tink-leader-election-rolebinding nodeSelector: {} + # singleNodeClusterConfig to add tolerations for deployments on control plane nodes. This is defaulted to false. + singleNodeClusterConfig: + controlPlaneTolerationsEnabled: false + weight: 1 server: deploy: true @@ -39,3 +43,7 @@ server: roleName: tink-server-role roleBindingName: tink-server-rolebinding nodeSelector: {} + # singleNodeClusterConfig to add tolerations for deployments on control plane nodes. This is defaulted to false. + singleNodeClusterConfig: + controlPlaneTolerationsEnabled: false + nodeAffinityWeight: 1