feat(t8s-cluster/clusterClass)!: configure static machineDeploymentCl…

…asses instead of one per computePlane Closes #584 Needs https://gitlab.teuto.net/4teuto/ops/k8s/t8s-engine/-/merge_requests/661
teutonet · Oct 12, 2023 · 2795146 · 2795146
1 parent f94fc7a
commit 2795146
Show file tree

Hide file tree

Showing 25 changed files with 122 additions and 103 deletions.
diff --git a/charts/t8s-cluster/ci/gpu-flavor-values.yaml b/charts/t8s-cluster/ci/gpu-flavor-values.yaml
@@ -1,4 +1,4 @@
-workers:
+nodePools:
   test:
     replicas: 1
     availabilityZone: Zone1

diff --git a/charts/t8s-cluster/ci/injectedCertificateAuthorities-values.yaml b/charts/t8s-cluster/ci/injectedCertificateAuthorities-values.yaml
@@ -46,7 +46,7 @@ global:
     -----END CERTIFICATE-----
 controlPlane:
   flavor: standard.2.1905
-workers:
+nodePools:
   test:
     replicas: 1
     availabilityZone: Zone1

diff --git a/charts/t8s-cluster/ci/securityGroups-values.yaml b/charts/t8s-cluster/ci/securityGroups-values.yaml
@@ -3,7 +3,7 @@ controlPlane:
   additionalSecurityGroups:
     - kuchen
     - pizza
-workers:
+nodePools:
   test:
     replicas: 1
     availabilityZone: Zone1

diff --git a/charts/t8s-cluster/templates/_helpers.tpl b/charts/t8s-cluster/templates/_helpers.tpl
@@ -9,7 +9,8 @@
 {{- end -}}
 
 {{- define "t8s-cluster.helm.chartVersion" -}}
-{{- dig .repo "charts" .chart nil .context.Values.global.helmRepositories | required (printf "The repo '%s' is either missing or doesn't contain the chart '%s'" .repo .chart) -}}
+  {{- $_ := set . "Values" .context.Values -}}
+  {{- dig .repo "charts" .chart nil .Values.global.helmRepositories | required (printf "The repo '%s' is either missing or doesn't contain the chart '%s'" .repo .chart) -}}
 {{- end -}}
 
 {{- define "t8s-cluster.helm.chartSpec" -}}
@@ -22,11 +23,12 @@ sourceRef:
 {{- end -}}
 
 {{- define "t8s-cluster.hasGPUNodes" -}}
-{{- $hasGPUFlavor := false -}}
-{{- range $name, $machineDeploymentClass := .Values.workers -}}
-  {{- if contains "gpu" (lower $machineDeploymentClass.flavor) -}}
-    {{- $hasGPUFlavor = true -}}
+  {{- $_ := set . "Values" .context.Values -}}
+  {{- $hasGPUFlavor := false -}}
+  {{- range $name, $machineDeploymentClass := .Values.nodePools -}}
+    {{- if contains "gpu" (lower $machineDeploymentClass.flavor) -}}
+      {{- $hasGPUFlavor = true -}}
+    {{- end -}}
   {{- end -}}
-{{- end -}}
-{{- $hasGPUFlavor -}}
+  {{- $hasGPUFlavor -}}
 {{- end -}}
diff --git a/charts/t8s-cluster/templates/management-cluster/cluster.yaml b/charts/t8s-cluster/templates/management-cluster/cluster.yaml
@@ -1,4 +1,4 @@
-apiVersion: t8s-engine.teuto.net/v1alpha1
+apiVersion: t8s-engine.teuto.net/v1beta1
 kind: TeutonetesCluster
 metadata:
   name: {{ .Release.Name }}
@@ -28,7 +28,7 @@ spec:
   version: {{ printf "v%d.%d.%d" (.major | int) (.minor | int) (.patch | int) }}
   {{- end }}
   clusterClass: {{ $.Release.Name }}
-  workers: {{- range $name, $machineDeploymentClass := .Values.workers }}
+  nodePools: {{- range $name, $machineDeploymentClass := .Values.nodePools }}
     {{ $name -}}:
       {{- if $machineDeploymentClass.availabilityZone }}
       availabilityZone: {{ $machineDeploymentClass.availabilityZone | quote }}

diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl
@@ -7,6 +7,7 @@ openstack
 {{- end -}}
 
 {{- define "t8s-cluster.clusterClass.getIdentityRefSecretName" -}}
+  {{- $_ := set . "Release" .context.Release -}}
   {{- printf "cloud-config-%s" .Release.Name -}}
 {{- end -}}
 
@@ -16,6 +17,7 @@ openstack
 {{- end -}}
 
 {{- define "t8s-cluster.clusterClass.preKubeadmCommands" -}}
+  {{- $_ := set . "Values" .context.Values -}}
   {{- $commands := list -}}
   {{- $commands = append $commands "bash /etc/kube-proxy-patch.sh" }}
   {{- if .Values.global.injectedCertificateAuthorities -}}

diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml
@@ -1,3 +1,4 @@
+{{- $machineDeploymentClasses := dict "compute-plane" false "gpu-compute-plane" true }}
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: ClusterClass
 metadata:
@@ -19,120 +20,122 @@ spec:
     nodeDrainTimeout: {{ .Values.controlPlane.nodeDrainTimeout | default "8m" }}
     machineInfrastructure:
       ref:
-        apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+        apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
         kind: OpenStackMachineTemplate
         name: {{ printf "%s-control-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfControlPlane" (dict "context" $)) }}
     ref:
       apiVersion: controlplane.cluster.x-k8s.io/v1beta1
       kind: KubeadmControlPlaneTemplate
-      # the full context is needed for .Files.Get
-      name: {{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }}
+      name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }}
   infrastructure:
     ref:
-      apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+      apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
       kind: OpenStackClusterTemplate
       name: {{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackClusterTemplate.specHash" (dict "context" $)) }}
   variables:
     - name: controlPlaneServerGroupID
       required: true
       schema:
         openAPIV3Schema:
-          description: |-
-            OpenStack Server Group to use for Control Plane machines.
-            Field is optional, but must be set for HA clusters.
           type: string
     - name: machineDeploymentServerGroupID
       required: false
       schema:
         openAPIV3Schema:
-          description: |-
-            OpenStack Server Group to use for MachineDeployment machines.
-            Field is optional, but should be set for HA clusters.
           type: string
     - name: dnsNameservers
       required: true
       schema:
         openAPIV3Schema:
-          description: |
-            [Optional] Can be specified to set a list of DNS Nameservers for the servers.
           type: array
           minItems: 1
           uniqueItems: true
           items:
             type: string
             format: ipv4
-            example: "203.0.113.123"
     - name: controlPlaneAvailabilityZones
       required: false
       schema:
         openAPIV3Schema:
           type: array
-          description: |
-            [Optional] Can be specified to restrict the Zones used for
-                  ControlPlane Machines (default is all available Zones)
           minItems: 1
           uniqueItems: true
           items:
             type: string
-            example: Zone1
+    - name: machineDeploymentFlavor
+      required: true
+      schema:
+        openAPIV3Schema:
+          type: string
   patches:
-    - definitions:
+    - name: controlPlaneServerGroupID
+      description: Sets the ServerGroupID for Control Plane machines.
+      definitions:
         - jsonPatches:
             - op: add
               path: /spec/template/spec/serverGroupID
               valueFrom:
                 variable: controlPlaneServerGroupID
           selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
             kind: OpenStackMachineTemplate
             matchResources:
               controlPlane: true
-      description: Sets the ServerGroupID for Control Plane machines.
-      name: controlPlaneServerGroupID
-    - definitions:
+    - name: machineDeploymentServerGroupID
+      description: Sets the ServerGroupID for MachineDeployment machines.
+      definitions:
         - jsonPatches:
             - op: add
               path: /spec/template/spec/serverGroupID
               valueFrom:
                 variable: machineDeploymentServerGroupID
           selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
             kind: OpenStackMachineTemplate
             matchResources:
               machineDeploymentClass:
-                names: {{- .Values.workers | keys | sortAlpha | toYaml | nindent 18 }}
-      description: Sets the ServerGroupID for MachineDeployment machines.
-      name: machineDeploymentServerGroupID
-    - definitions:
+                names: {{- $machineDeploymentClasses | keys | sortAlpha | toYaml | nindent 18 }}
+    - name: machineDeploymentFlavour
+      definitions:
         - jsonPatches:
             - op: add
-              path: /spec/template/spec/image
+              path: /spec/template/spec/flavor
               valueFrom:
-                template: {{ .Values.imageNameTemplate.workers }}
+                variable: machineDeploymentFlavor
           selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
             kind: OpenStackMachineTemplate
             matchResources:
               machineDeploymentClass:
-                names: {{- .Values.workers | keys | sortAlpha | toYaml | nindent 18 }}
+                names: {{- $machineDeploymentClasses | keys | sortAlpha | toYaml | nindent 18 }}
+    - name: imageVersion
+      description: Sets the image version for machines.
+      definitions:
         - jsonPatches:
-            - op: add
+            - &imagePatch
+              op: add
               path: /spec/template/spec/image
+              valueFrom:
+                template: {{ .Values.imageNameTemplate.computePlane }}
+          selector: &imagePatchSelector
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
+            kind: OpenStackMachineTemplate
+            matchResources:
+              machineDeploymentClass:
+                names: {{- $machineDeploymentClasses | keys | sortAlpha | toYaml | nindent 18 }}
+        - jsonPatches:
+            - <<: *imagePatch
               valueFrom:
                 template: {{ .Values.imageNameTemplate.controlPlane }}
           selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
-            kind: OpenStackMachineTemplate
+            <<: *imagePatchSelector
             matchResources:
               controlPlane: true
-
-      description: Sets the image version for machines.
-      name: imageVersion
     - name: controlPlaneAvailabilityZones
       enabledIf: {{ `{{ if .controlPlaneAvailabilityZones }}true{{ end }}` | quote }}
       definitions:
         - selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
             kind: OpenStackClusterTemplate
             matchResources:
               infrastructureCluster: true
@@ -144,7 +147,7 @@ spec:
     - name: dnsNameservers
       definitions:
         - selector:
-            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+            apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
             kind: OpenStackClusterTemplate
             matchResources:
               infrastructureCluster: true
@@ -155,8 +158,7 @@ spec:
                 variable: dnsNameservers
   workers:
     machineDeployments:
-      {{- range $name, $machineDeploymentClass := .Values.workers }}
-        {{- $isGpuDeploymentClass := $machineDeploymentClass.flavor | contains "gpu" }}
+      {{- range $name, $isGpuDeploymentClass := $machineDeploymentClasses }}
       - class: {{ $name }}
         machineHealthCheck:
           nodeStartupTimeout: 8m
@@ -167,7 +169,7 @@ spec:
             - status: 'False'
               timeout: 300s
               type: Ready
-        nodeDrainTimeout: {{ $machineDeploymentClass.nodeDrainTimeout | default "8m" }}
+        nodeDrainTimeout: "8m"
         strategy:
           type: RollingUpdate
           rollingUpdate:
@@ -184,10 +186,10 @@ spec:
             ref:
               apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
               kind: KubeadmConfigTemplate
-              name: {{ printf "%s-%s-worker" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }}
+              name: {{ printf "%s-%s-compute-plane" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }}
           infrastructure:
             ref:
-              apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }}
+              apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }}
               kind: OpenStackMachineTemplate
-              name: {{ printf "%s-%s-%s" $.Release.Name $name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfWorkers" (dict "context" $ "worker" $machineDeploymentClass)) }}
+              name: {{ printf "%s-compute-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfNodePools" (dict "name" "compute-plane" "context" $)) }}
       {{- end }}
diff --git a/...tes/management-cluster/clusterClass/kubeadmConfigTemplate/_kubeadmConfigTemplateSpec.yaml b/...tes/management-cluster/clusterClass/kubeadmConfigTemplate/_kubeadmConfigTemplateSpec.yaml
@@ -2,11 +2,11 @@
   {{- $_ := set . "Values" .context.Values -}}
 joinConfiguration:
   nodeRegistration:
-    kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" .context | nindent 6 }}
+    kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 6 }}
     name: '{{ `{{ local_hostname }}` }}'
   patches:
     directory: /etc/kubernetes/patches
-files: {{- include "t8s-cluster.patches.kubelet.patches" .context | nindent 2 }}
+files: {{- include "t8s-cluster.patches.kubelet.patches" (dict "context" .context) | nindent 2 }}
     {{- if .Values.containerRegistryMirror.mirrorEndpoint }}
       {{- include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" .context) | nindent 2 }}
     {{- end }}

diff --git a/...emplate/_workerKubeadmConfigTemplate.yaml → ...plate/_nodePoolKubeadmConfigTemplate.yaml b/...emplate/_workerKubeadmConfigTemplate.yaml → ...plate/_nodePoolKubeadmConfigTemplate.yaml
@@ -2,7 +2,7 @@
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfigTemplate
 metadata:
-  name: {{ printf "%s-%s-worker" .context.Release.Name (.gpu | ternary "gpu" "standard") }}
+  name: {{ printf "%s-%s-compute-plane" .context.Release.Name (.gpu | ternary "gpu" "standard") }}
   namespace: {{ .context.Release.Namespace }}
   labels: {{- include "common.labels.standard" .context | nindent 4 }}
 spec:

diff --git a/...plate/gpuWorkerKubeadmConfigTemplate.yaml → ...ate/gpuNodePoolKubeadmConfigTemplate.yaml b/...plate/gpuWorkerKubeadmConfigTemplate.yaml → ...ate/gpuNodePoolKubeadmConfigTemplate.yaml
@@ -1,3 +1,3 @@
-{{- if eq (include "t8s-cluster.hasGPUNodes" .) "true" }}
+{{- if eq (include "t8s-cluster.hasGPUNodes" (dict "context" $)) "true" }}
   {{- include "t8s-cluster.clusterClass.kubeadmConfigTemplate" (dict "gpu" true "context" $) -}}
 {{- end }}
diff --git a/.../standardWorkerKubeadmConfigTemplate.yaml → ...tandardNodePoolKubeadmConfigTemplate.yaml b/.../standardWorkerKubeadmConfigTemplate.yaml → ...tandardNodePoolKubeadmConfigTemplate.yaml
@@ -1,5 +1,5 @@
 {{- $hasStandardFlavor := false -}}
-{{- range $name, $machineDeploymentClass := .Values.workers }}
+{{- range $name, $machineDeploymentClass := .Values.nodePools }}
   {{- if not (contains "gpu" (lower $machineDeploymentClass.flavor)) -}}
     {{- $hasStandardFlavor = true -}}
   {{- end -}}

diff --git a/...uster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_helpers.tpl b/...uster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_helpers.tpl
@@ -1,6 +1,7 @@
 {{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" -}}
+  {{/* the full context is needed for .Files.Get */}}
   {{- $inputs := (dict
-    "spec" (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" .)
+    "spec" (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" $)
     ) -}}
   {{- mustToJson $inputs | toString | quote | sha1sum | trunc 8 -}}
 {{- end -}}
diff --git a/...t-cluster/clusterClass/kubeadmnControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml b/...t-cluster/clusterClass/kubeadmnControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml
@@ -1,6 +1,7 @@
 {{/*
 KubeadmControlPlaneTemplate is immutable. We need to create new versions during upgrades.
 Here we are generating a hash suffix.
+This function needs the whole `$` context to be able to use `.Files.Get`
 */}}
 {{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" -}}
 clusterConfiguration:
@@ -36,7 +37,7 @@ clusterConfiguration:
       authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics
       bind-address: 0.0.0.0
       profiling: 'false'
-files: {{- include "t8s-cluster.patches.kubelet.patches" $ | nindent 2 }}
+files: {{- include "t8s-cluster.patches.kubelet.patches" (dict "context" $) | nindent 2 }}
   - content: |- {{- .Files.Get "files/admission-control-config.yaml" | nindent 6 }}
     path: *admissionControlConfigFilePath
   - content: |- {{- .Files.Get "files/event-rate-limit-config.yaml" | nindent 6 }}
@@ -57,16 +58,16 @@ files: {{- include "t8s-cluster.patches.kubelet.patches" $ | nindent 2 }}
   {{- end }}
 initConfiguration:
   nodeRegistration:
-    kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" $ | nindent 6 }}
+    kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 6 }}
     name: '{{ `{{ local_hostname }}` }}'
   patches:
     directory: {{ include "t8s-cluster.patches.directory" (dict) }}
 joinConfiguration:
   nodeRegistration:
-    kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" $ | nindent 6 }}
+    kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 6 }}
     name: '{{ `{{ local_hostname }}` }}'
   patches:
     directory: {{ include "t8s-cluster.patches.directory" (dict) }}
-preKubeadmCommands: {{- include "t8s-cluster.clusterClass.preKubeadmCommands" $ | nindent 2 }}
+preKubeadmCommands: {{- include "t8s-cluster.clusterClass.preKubeadmCommands" (dict "context" $) | nindent 2 }}
 postKubeadmCommands: {{- include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | nindent 2 }}
 {{- end -}}
diff --git a/...gement-cluster/clusterClass/kubeadmnControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml b/...gement-cluster/clusterClass/kubeadmnControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml
@@ -1,11 +1,10 @@
 apiVersion: controlplane.cluster.x-k8s.io/v1beta1
 kind: KubeadmControlPlaneTemplate
 metadata:
-  name: {{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }}
+  name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }}
   namespace: {{ $.Release.Namespace }}
   labels: {{- include "common.labels.standard" $ | nindent 4 }}
 spec:
   template:
     spec:
-      # the full context is needed for .Files.Get
-      kubeadmConfigSpec: {{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" . | nindent 8 }}
+      kubeadmConfigSpec: {{/* the full context is needed for .Files.Get */}}{{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" . | nindent 8 }}
diff --git a/...s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_helpers.tpl b/...s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_helpers.tpl
@@ -1,6 +1,6 @@
 {{- define "t8s-cluster.clusterClass.openStackClusterTemplate.specHash" -}}
   {{- $inputs := (dict
-    "spec" (include "t8s-cluster.clusterClass.openStackClusterTemplate.spec" .)
+    "spec" (include "t8s-cluster.clusterClass.openStackClusterTemplate.spec" (dict "context" .context))
     "infrastructureApiVersion" (include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict))
     ) -}}
   {{- mustToJson $inputs | toString | quote | sha1sum | trunc 8 -}}