diff --git a/charts/t8s-cluster/ci/gpu-flavor-values.yaml b/charts/t8s-cluster/ci/gpu-flavor-values.yaml index 0551177a46..4f0d7a3a69 100644 --- a/charts/t8s-cluster/ci/gpu-flavor-values.yaml +++ b/charts/t8s-cluster/ci/gpu-flavor-values.yaml @@ -1,4 +1,4 @@ -workers: +nodePools: test: replicas: 1 availabilityZone: Zone1 diff --git a/charts/t8s-cluster/ci/injectedCertificateAuthorities-values.yaml b/charts/t8s-cluster/ci/injectedCertificateAuthorities-values.yaml index d7f5f989a5..94452dd8b9 100644 --- a/charts/t8s-cluster/ci/injectedCertificateAuthorities-values.yaml +++ b/charts/t8s-cluster/ci/injectedCertificateAuthorities-values.yaml @@ -46,7 +46,7 @@ global: -----END CERTIFICATE----- controlPlane: flavor: standard.2.1905 -workers: +nodePools: test: replicas: 1 availabilityZone: Zone1 diff --git a/charts/t8s-cluster/ci/securityGroups-values.yaml b/charts/t8s-cluster/ci/securityGroups-values.yaml index 9ba6683454..1ebd872521 100644 --- a/charts/t8s-cluster/ci/securityGroups-values.yaml +++ b/charts/t8s-cluster/ci/securityGroups-values.yaml @@ -3,7 +3,7 @@ controlPlane: additionalSecurityGroups: - kuchen - pizza -workers: +nodePools: test: replicas: 1 availabilityZone: Zone1 diff --git a/charts/t8s-cluster/templates/_helpers.tpl b/charts/t8s-cluster/templates/_helpers.tpl index f96ce5b3c3..d57b0c45e3 100644 --- a/charts/t8s-cluster/templates/_helpers.tpl +++ b/charts/t8s-cluster/templates/_helpers.tpl @@ -9,7 +9,8 @@ {{- end -}} {{- define "t8s-cluster.helm.chartVersion" -}} -{{- dig .repo "charts" .chart nil .context.Values.global.helmRepositories | required (printf "The repo '%s' is either missing or doesn't contain the chart '%s'" .repo .chart) -}} + {{- $_ := set . "Values" .context.Values -}} + {{- dig .repo "charts" .chart nil .Values.global.helmRepositories | required (printf "The repo '%s' is either missing or doesn't contain the chart '%s'" .repo .chart) -}} {{- end -}} {{- define "t8s-cluster.helm.chartSpec" -}} @@ -22,11 +23,12 @@ sourceRef: {{- end -}} {{- define "t8s-cluster.hasGPUNodes" -}} -{{- $hasGPUFlavor := false -}} -{{- range $name, $machineDeploymentClass := .Values.workers -}} - {{- if contains "gpu" (lower $machineDeploymentClass.flavor) -}} - {{- $hasGPUFlavor = true -}} + {{- $_ := set . "Values" .context.Values -}} + {{- $hasGPUFlavor := false -}} + {{- range $name, $machineDeploymentClass := .Values.nodePools -}} + {{- if contains "gpu" (lower $machineDeploymentClass.flavor) -}} + {{- $hasGPUFlavor = true -}} + {{- end -}} {{- end -}} -{{- end -}} -{{- $hasGPUFlavor -}} + {{- $hasGPUFlavor -}} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/cluster.yaml b/charts/t8s-cluster/templates/management-cluster/cluster.yaml index 1340bb6f2b..6d2efb85fd 100644 --- a/charts/t8s-cluster/templates/management-cluster/cluster.yaml +++ b/charts/t8s-cluster/templates/management-cluster/cluster.yaml @@ -1,4 +1,4 @@ -apiVersion: t8s-engine.teuto.net/v1alpha1 +apiVersion: t8s-engine.teuto.net/v1beta1 kind: TeutonetesCluster metadata: name: {{ .Release.Name }} @@ -28,7 +28,7 @@ spec: version: {{ printf "v%d.%d.%d" (.major | int) (.minor | int) (.patch | int) }} {{- end }} clusterClass: {{ $.Release.Name }} - workers: {{- range $name, $machineDeploymentClass := .Values.workers }} + nodePools: {{- range $name, $machineDeploymentClass := .Values.nodePools }} {{ $name -}}: {{- if $machineDeploymentClass.availabilityZone }} availabilityZone: {{ $machineDeploymentClass.availabilityZone | quote }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl index 23a04bbd87..3c0f2093e3 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl @@ -7,6 +7,7 @@ openstack {{- end -}} {{- define "t8s-cluster.clusterClass.getIdentityRefSecretName" -}} + {{- $_ := set . "Release" .context.Release -}} {{- printf "cloud-config-%s" .Release.Name -}} {{- end -}} @@ -16,6 +17,7 @@ openstack {{- end -}} {{- define "t8s-cluster.clusterClass.preKubeadmCommands" -}} + {{- $_ := set . "Values" .context.Values -}} {{- $commands := list -}} {{- $commands = append $commands "bash /etc/kube-proxy-patch.sh" }} {{- if .Values.global.injectedCertificateAuthorities -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml index c7cd52b72b..3f3f2473e6 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml @@ -1,3 +1,4 @@ +{{- $machineDeploymentClasses := dict "compute-plane" false "gpu-compute-plane" true }} apiVersion: cluster.x-k8s.io/v1beta1 kind: ClusterClass metadata: @@ -19,17 +20,16 @@ spec: nodeDrainTimeout: {{ .Values.controlPlane.nodeDrainTimeout | default "8m" }} machineInfrastructure: ref: - apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} + apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackMachineTemplate name: {{ printf "%s-control-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfControlPlane" (dict "context" $)) }} ref: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate - # the full context is needed for .Files.Get - name: {{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }} + name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }} infrastructure: ref: - apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} + apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackClusterTemplate name: {{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackClusterTemplate.specHash" (dict "context" $)) }} variables: @@ -37,102 +37,105 @@ spec: required: true schema: openAPIV3Schema: - description: |- - OpenStack Server Group to use for Control Plane machines. - Field is optional, but must be set for HA clusters. type: string - name: machineDeploymentServerGroupID required: false schema: openAPIV3Schema: - description: |- - OpenStack Server Group to use for MachineDeployment machines. - Field is optional, but should be set for HA clusters. type: string - name: dnsNameservers required: true schema: openAPIV3Schema: - description: | - [Optional] Can be specified to set a list of DNS Nameservers for the servers. type: array minItems: 1 uniqueItems: true items: type: string format: ipv4 - example: "203.0.113.123" - name: controlPlaneAvailabilityZones required: false schema: openAPIV3Schema: type: array - description: | - [Optional] Can be specified to restrict the Zones used for - ControlPlane Machines (default is all available Zones) minItems: 1 uniqueItems: true items: type: string - example: Zone1 + - name: machineDeploymentFlavor + required: true + schema: + openAPIV3Schema: + type: string patches: - - definitions: + - name: controlPlaneServerGroupID + description: Sets the ServerGroupID for Control Plane machines. + definitions: - jsonPatches: - op: add path: /spec/template/spec/serverGroupID valueFrom: variable: controlPlaneServerGroupID selector: - apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} + apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackMachineTemplate matchResources: controlPlane: true - description: Sets the ServerGroupID for Control Plane machines. - name: controlPlaneServerGroupID - - definitions: + - name: machineDeploymentServerGroupID + description: Sets the ServerGroupID for MachineDeployment machines. + definitions: - jsonPatches: - op: add path: /spec/template/spec/serverGroupID valueFrom: variable: machineDeploymentServerGroupID selector: - apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} + apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackMachineTemplate matchResources: machineDeploymentClass: - names: {{- .Values.workers | keys | sortAlpha | toYaml | nindent 18 }} - description: Sets the ServerGroupID for MachineDeployment machines. - name: machineDeploymentServerGroupID - - definitions: + names: {{- $machineDeploymentClasses | keys | sortAlpha | toYaml | nindent 18 }} + - name: machineDeploymentFlavour + definitions: - jsonPatches: - op: add - path: /spec/template/spec/image + path: /spec/template/spec/flavor valueFrom: - template: {{ .Values.imageNameTemplate.workers }} + variable: machineDeploymentFlavor selector: - apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} + apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackMachineTemplate matchResources: machineDeploymentClass: - names: {{- .Values.workers | keys | sortAlpha | toYaml | nindent 18 }} + names: {{- $machineDeploymentClasses | keys | sortAlpha | toYaml | nindent 18 }} + - name: imageVersion + description: Sets the image version for machines. + definitions: - jsonPatches: - - op: add + - &imagePatch + op: add path: /spec/template/spec/image + valueFrom: + template: {{ .Values.imageNameTemplate.computePlane }} + selector: &imagePatchSelector + apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} + kind: OpenStackMachineTemplate + matchResources: + machineDeploymentClass: + names: {{- $machineDeploymentClasses | keys | sortAlpha | toYaml | nindent 18 }} + - jsonPatches: + - <<: *imagePatch valueFrom: template: {{ .Values.imageNameTemplate.controlPlane }} selector: - apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} - kind: OpenStackMachineTemplate + <<: *imagePatchSelector matchResources: controlPlane: true - - description: Sets the image version for machines. - name: imageVersion - name: controlPlaneAvailabilityZones enabledIf: {{ `{{ if .controlPlaneAvailabilityZones }}true{{ end }}` | quote }} definitions: - selector: - apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} + apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackClusterTemplate matchResources: infrastructureCluster: true @@ -144,7 +147,7 @@ spec: - name: dnsNameservers definitions: - selector: - apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} + apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackClusterTemplate matchResources: infrastructureCluster: true @@ -155,8 +158,7 @@ spec: variable: dnsNameservers workers: machineDeployments: - {{- range $name, $machineDeploymentClass := .Values.workers }} - {{- $isGpuDeploymentClass := $machineDeploymentClass.flavor | contains "gpu" }} + {{- range $name, $isGpuDeploymentClass := $machineDeploymentClasses }} - class: {{ $name }} machineHealthCheck: nodeStartupTimeout: 8m @@ -167,7 +169,7 @@ spec: - status: 'False' timeout: 300s type: Ready - nodeDrainTimeout: {{ $machineDeploymentClass.nodeDrainTimeout | default "8m" }} + nodeDrainTimeout: "8m" strategy: type: RollingUpdate rollingUpdate: @@ -184,10 +186,10 @@ spec: ref: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate - name: {{ printf "%s-%s-worker" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }} + name: {{ printf "%s-%s-compute-plane" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }} infrastructure: ref: - apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} + apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackMachineTemplate - name: {{ printf "%s-%s-%s" $.Release.Name $name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfWorkers" (dict "context" $ "worker" $machineDeploymentClass)) }} + name: {{ printf "%s-compute-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfNodePools" (dict "name" "compute-plane" "context" $)) }} {{- end }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_kubeadmConfigTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_kubeadmConfigTemplateSpec.yaml index 61c7494966..14713ce86d 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_kubeadmConfigTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_kubeadmConfigTemplateSpec.yaml @@ -2,11 +2,11 @@ {{- $_ := set . "Values" .context.Values -}} joinConfiguration: nodeRegistration: - kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" .context | nindent 6 }} + kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 6 }} name: '{{ `{{ local_hostname }}` }}' patches: directory: /etc/kubernetes/patches -files: {{- include "t8s-cluster.patches.kubelet.patches" .context | nindent 2 }} +files: {{- include "t8s-cluster.patches.kubelet.patches" (dict "context" .context) | nindent 2 }} {{- if .Values.containerRegistryMirror.mirrorEndpoint }} {{- include "t8s-cluster.clusterClass.containerdConfig.containerRegistryMirrorConfigs" (dict "context" .context) | nindent 2 }} {{- end }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_workerKubeadmConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_nodePoolKubeadmConfigTemplate.yaml similarity index 81% rename from charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_workerKubeadmConfigTemplate.yaml rename to charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_nodePoolKubeadmConfigTemplate.yaml index 0a6471f31f..9791e17e3f 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_workerKubeadmConfigTemplate.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/_nodePoolKubeadmConfigTemplate.yaml @@ -2,7 +2,7 @@ apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: - name: {{ printf "%s-%s-worker" .context.Release.Name (.gpu | ternary "gpu" "standard") }} + name: {{ printf "%s-%s-compute-plane" .context.Release.Name (.gpu | ternary "gpu" "standard") }} namespace: {{ .context.Release.Namespace }} labels: {{- include "common.labels.standard" .context | nindent 4 }} spec: diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/gpuWorkerKubeadmConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/gpuNodePoolKubeadmConfigTemplate.yaml similarity index 59% rename from charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/gpuWorkerKubeadmConfigTemplate.yaml rename to charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/gpuNodePoolKubeadmConfigTemplate.yaml index a5e88977b8..20fce6b20a 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/gpuWorkerKubeadmConfigTemplate.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/gpuNodePoolKubeadmConfigTemplate.yaml @@ -1,3 +1,3 @@ -{{- if eq (include "t8s-cluster.hasGPUNodes" .) "true" }} +{{- if eq (include "t8s-cluster.hasGPUNodes" (dict "context" $)) "true" }} {{- include "t8s-cluster.clusterClass.kubeadmConfigTemplate" (dict "gpu" true "context" $) -}} {{- end }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/standardWorkerKubeadmConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/standardNodePoolKubeadmConfigTemplate.yaml similarity index 82% rename from charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/standardWorkerKubeadmConfigTemplate.yaml rename to charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/standardNodePoolKubeadmConfigTemplate.yaml index d5d1bc3c64..ec2c5b129c 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/standardWorkerKubeadmConfigTemplate.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmConfigTemplate/standardNodePoolKubeadmConfigTemplate.yaml @@ -1,5 +1,5 @@ {{- $hasStandardFlavor := false -}} -{{- range $name, $machineDeploymentClass := .Values.workers }} +{{- range $name, $machineDeploymentClass := .Values.nodePools }} {{- if not (contains "gpu" (lower $machineDeploymentClass.flavor)) -}} {{- $hasStandardFlavor = true -}} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_helpers.tpl index 631d143f08..2723427816 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_helpers.tpl @@ -1,6 +1,7 @@ {{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" -}} + {{/* the full context is needed for .Files.Get */}} {{- $inputs := (dict - "spec" (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" .) + "spec" (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" $) ) -}} {{- mustToJson $inputs | toString | quote | sha1sum | trunc 8 -}} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml index 100c6c272b..6c62dfc57d 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml @@ -1,6 +1,7 @@ {{/* KubeadmControlPlaneTemplate is immutable. We need to create new versions during upgrades. Here we are generating a hash suffix. +This function needs the whole `$` context to be able to use `.Files.Get` */}} {{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" -}} clusterConfiguration: @@ -36,7 +37,7 @@ clusterConfiguration: authorization-always-allow-paths: /healthz,/readyz,/livez,/metrics bind-address: 0.0.0.0 profiling: 'false' -files: {{- include "t8s-cluster.patches.kubelet.patches" $ | nindent 2 }} +files: {{- include "t8s-cluster.patches.kubelet.patches" (dict "context" $) | nindent 2 }} - content: |- {{- .Files.Get "files/admission-control-config.yaml" | nindent 6 }} path: *admissionControlConfigFilePath - content: |- {{- .Files.Get "files/event-rate-limit-config.yaml" | nindent 6 }} @@ -57,16 +58,16 @@ files: {{- include "t8s-cluster.patches.kubelet.patches" $ | nindent 2 }} {{- end }} initConfiguration: nodeRegistration: - kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" $ | nindent 6 }} + kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 6 }} name: '{{ `{{ local_hostname }}` }}' patches: directory: {{ include "t8s-cluster.patches.directory" (dict) }} joinConfiguration: nodeRegistration: - kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" $ | nindent 6 }} + kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict) | nindent 6 }} name: '{{ `{{ local_hostname }}` }}' patches: directory: {{ include "t8s-cluster.patches.directory" (dict) }} -preKubeadmCommands: {{- include "t8s-cluster.clusterClass.preKubeadmCommands" $ | nindent 2 }} +preKubeadmCommands: {{- include "t8s-cluster.clusterClass.preKubeadmCommands" (dict "context" $) | nindent 2 }} postKubeadmCommands: {{- include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | nindent 2 }} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml index d739ed8fb1..3ef2fd51c8 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmnControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml @@ -1,11 +1,10 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate metadata: - name: {{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }} + name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }} namespace: {{ $.Release.Namespace }} labels: {{- include "common.labels.standard" $ | nindent 4 }} spec: template: spec: - # the full context is needed for .Files.Get - kubeadmConfigSpec: {{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" . | nindent 8 }} \ No newline at end of file + kubeadmConfigSpec: {{/* the full context is needed for .Files.Get */}}{{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" . | nindent 8 }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_helpers.tpl index 15af9d73af..c18b428d23 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_helpers.tpl @@ -1,6 +1,6 @@ {{- define "t8s-cluster.clusterClass.openStackClusterTemplate.specHash" -}} {{- $inputs := (dict - "spec" (include "t8s-cluster.clusterClass.openStackClusterTemplate.spec" .) + "spec" (include "t8s-cluster.clusterClass.openStackClusterTemplate.spec" (dict "context" .context)) "infrastructureApiVersion" (include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict)) ) -}} {{- mustToJson $inputs | toString | quote | sha1sum | trunc 8 -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml index 9310806a47..d37498339c 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml @@ -17,14 +17,14 @@ bastion: flavor: standard.1.1905 identityRef: kind: Secret - name: {{ include "t8s-cluster.clusterClass.getIdentityRefSecretName" .context }} + name: {{ include "t8s-cluster.clusterClass.getIdentityRefSecretName" (dict "context" .context) }} image: Ubuntu 20.04 # artifacthub-ignore sshKeyName: {{ .Values.bastion.sshKeyName }} cloudName: {{ include "t8s-cluster.clusterClass.cloudName" (dict) }} disablePortSecurity: false identityRef: kind: Secret - name: {{ include "t8s-cluster.clusterClass.getIdentityRefSecretName" .context }} + name: {{ include "t8s-cluster.clusterClass.getIdentityRefSecretName" (dict "context" .context) }} managedSecurityGroups: true nodeCidr: 10.6.0.0/24 {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/openStackClusterTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/openStackClusterTemplate.yaml index d9ee4d1793..e9bc712fb8 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/openStackClusterTemplate.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/openStackClusterTemplate.yaml @@ -1,7 +1,7 @@ {{- if false }} apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 {{- else }} -apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} +apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} {{- end }} kind: OpenStackClusterTemplate metadata: diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_helpers.tpl index 3904ae7a89..518f66dfa6 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_helpers.tpl @@ -1,21 +1,28 @@ {{- define "t8s-cluster.clusterClass.openStackMachineTemplate.specHash" -}} {{- $inputs := (dict - "spec" (include "t8s-cluster.clusterClass.openStackMachineTemplate.spec" (dict "machineDeploymentClass" .machineDeploymentClass "name" .name "context" .context)) + "spec" (include "t8s-cluster.clusterClass.openStackMachineTemplate.spec" (dict "name" .name "context" .context)) "infrastructureApiVersion" (include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict)) ) -}} {{- mustToJson $inputs | toString | quote | sha1sum | trunc 8 -}} {{- end -}} {{- define "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfControlPlane" -}} - {{- include "t8s-cluster.clusterClass.openStackMachineTemplate.specHash" (dict "machineDeploymentClass" .context.Values.controlPlane "name" "control-plane" "context" .context) -}} + {{- include "t8s-cluster.clusterClass.openStackMachineTemplate.specHash" (dict "name" "control-plane" "context" .context) -}} {{- end -}} -{{- define "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfWorkers" -}} - {{- include "t8s-cluster.clusterClass.openStackMachineTemplate.specHash" (dict "machineDeploymentClass" .worker "name" "worker" "context" .context) -}} +{{- define "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfNodePools" -}} + {{- include "t8s-cluster.clusterClass.openStackMachineTemplate.specHash" (dict "name" "compute-plane" "context" .context) -}} {{- end -}} {{- define "t8s-cluster.clusterClass.securityGroups" -}} - {{- $securityGroups := .additionalSecurityGroups | default (list) -}} + {{- $_ := set . "Values" .context.Values -}} + {{- $additionalSecurityGroups := list -}} + {{- if eq .name "control-plane" -}} + {{- $additionalSecurityGroups = .Values.controlPlane.additionalSecurityGroups -}} + {{- else -}} + {{- $additionalSecurityGroups = .Values.additionalComputePlaneSecurityGroups -}} + {{- end -}} + {{- $securityGroups := $additionalSecurityGroups | default (list) -}} {{- $securityGroups = append $securityGroups "default" | sortAlpha | uniq }} {{- $securityGroupsObject := list -}} {{- range $name := $securityGroups -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml index 592edc0599..85341c386e 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml @@ -5,11 +5,13 @@ Here we are generating a hash suffix. {{- define "t8s-cluster.clusterClass.openStackMachineTemplate.spec" -}} {{- $_ := set . "Values" .context.Values -}} cloudName: {{ include "t8s-cluster.clusterClass.cloudName" (dict) }} -flavor: {{ .machineDeploymentClass.flavor }} + {{- if eq .name "control-plane" }} +flavor: {{ .Values.controlPlane.flavor }} + {{- end }} identityRef: - name: {{ include "t8s-cluster.clusterClass.getIdentityRefSecretName" .context }} + name: {{ include "t8s-cluster.clusterClass.getIdentityRefSecretName" (dict "context" .context) }} kind: Secret -securityGroups: {{- include "t8s-cluster.clusterClass.securityGroups" .machineDeploymentClass | nindent 2 }} +securityGroups: {{- include "t8s-cluster.clusterClass.securityGroups" (dict "name" .name "context" .context) | nindent 2 }} {{- with .Values.sshKeyName }} sshKeyName: {{ . }} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/openStackMachineTemplates.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/openStackMachineTemplates.yaml index 86894b238c..098a8b7b47 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/openStackMachineTemplates.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/openStackMachineTemplates.yaml @@ -1,16 +1,16 @@ -{{- range $name, $machineDeploymentClass := (merge (deepCopy .Values.workers) (dict "control-plane" .Values.controlPlane)) }} +{{- range $name := list "compute-plane" "control-plane" }} {{- if false }} apiVersion: infrastructure.cluster.x-k8s.io/v1alpha7 {{- else }} -apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" $ }} +apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} {{- end }} kind: OpenStackMachineTemplate metadata: - name: {{ printf "%s-%s-%s" $.Release.Name $name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHash" (dict "machineDeploymentClass" $machineDeploymentClass "name" $name "context" $)) }} + name: {{ printf "%s-%s-%s" $.Release.Name $name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHash" (dict "name" $name "context" $)) }} namespace: {{ $.Release.Namespace }} labels: {{- include "common.labels.standard" $ | nindent 4 }} spec: template: - spec: {{- include "t8s-cluster.clusterClass.openStackMachineTemplate.spec" (dict "machineDeploymentClass" $machineDeploymentClass "context" $) | nindent 6 }} + spec: {{- include "t8s-cluster.clusterClass.openStackMachineTemplate.spec" (dict "name" $name "context" $) | nindent 6 }} --- {{- end }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl index 2e53994fcc..4e1c510f79 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl @@ -1,4 +1,5 @@ {{- define "t8s-cluster.patches.kubelet.imagePulls" -}} + {{- $_ := set . "Values" .context.Values -}} {{- include "t8s-cluster.patches.patchFile" (dict "values" (dict "serializeImagePulls" false "maxParallelImagePulls" .Values.global.kubeletExtraConfig.maxParallelImagePulls) "target" "kubeletconfiguration" "component" "imagePulls") -}} {{- end -}} @@ -13,9 +14,10 @@ {{- end -}} {{- define "t8s-cluster.patches.kubelet.patches" -}} - {{- $patches := list (include "t8s-cluster.patches.kubelet.default" . | fromYaml) -}} + {{- $_ := set . "Values" .context.Values -}} + {{- $patches := list (include "t8s-cluster.patches.kubelet.default" (dict) | fromYaml) -}} {{- if and (eq (int .Values.version.major) 1) (ge (int .Values.version.minor) 27) (gt (int .Values.global.kubeletExtraConfig.maxParallelImagePulls) 1) -}} - {{- $patches = append $patches (include "t8s-cluster.patches.kubelet.imagePulls" . | fromYaml) -}} + {{- $patches = append $patches (include "t8s-cluster.patches.kubelet.imagePulls" (dict "context" .context) | fromYaml) -}} {{- end -}} {{- $patches | toYaml -}} {{- end -}} diff --git a/charts/t8s-cluster/templates/workload-cluster/etcd-defrag.yaml b/charts/t8s-cluster/templates/workload-cluster/etcd-defrag.yaml index fc05a3cc77..b7802779ff 100644 --- a/charts/t8s-cluster/templates/workload-cluster/etcd-defrag.yaml +++ b/charts/t8s-cluster/templates/workload-cluster/etcd-defrag.yaml @@ -1,6 +1,7 @@ {{- include "t8s-cluster.helm.resourceIntoCluster" (dict "name" "etcd-defrag" "resource" (include "t8s-cluster.etcd-defrag" (dict "context" $)) "context" $ "additionalLabels" (dict "app.kubernetes.io/component" "etcd")) | nindent 0 }} {{- define "t8s-cluster.etcd-defrag" -}} + {{- $_ := set . "Values" .context.Values -}} apiVersion: batch/v1 kind: CronJob metadata: @@ -30,7 +31,7 @@ spec: value: "3" - name: ETCDCTL_ENDPOINTS value: 'localhost:2379' - image: {{ include "common.images.image" (dict "imageRoot" .context.Values.global.etcd.image "global" .context.Values.global) }} + image: {{ include "common.images.image" (dict "imageRoot" .Values.global.etcd.image "global" .Values.global) }} imagePullPolicy: IfNotPresent name: etcd-defrag volumeMounts: diff --git a/charts/t8s-cluster/templates/workload-cluster/gpu-operator.yaml b/charts/t8s-cluster/templates/workload-cluster/gpu-operator.yaml index 1db94791e8..77e47a65d2 100644 --- a/charts/t8s-cluster/templates/workload-cluster/gpu-operator.yaml +++ b/charts/t8s-cluster/templates/workload-cluster/gpu-operator.yaml @@ -1,4 +1,4 @@ -{{- if eq (include "t8s-cluster.hasGPUNodes" .) "true" }} +{{- if eq (include "t8s-cluster.hasGPUNodes" (dict "context" $)) "true" }} apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: @@ -45,7 +45,7 @@ spec: - key: node.kubernetes.io/instance-type operator: In {{- $gpuFlavours := list }} - {{- range $_, $machineDeploymentClass := .Values.workers -}} + {{- range $_, $machineDeploymentClass := .Values.nodePools -}} {{- if contains "gpu" (lower $machineDeploymentClass.flavor) -}} {{- $gpuFlavours = append $gpuFlavours $machineDeploymentClass.flavor -}} {{- end -}} diff --git a/charts/t8s-cluster/values.schema.json b/charts/t8s-cluster/values.schema.json index 8703347ac0..2264661fe4 100644 --- a/charts/t8s-cluster/values.schema.json +++ b/charts/t8s-cluster/values.schema.json @@ -165,7 +165,7 @@ ], "additionalProperties": false }, - "workers": { + "nodePools": { "type": "object", "additionalProperties": { "type": "object", @@ -180,9 +180,6 @@ "flavor": { "type": "string" }, - "additionalSecurityGroups": { - "$ref": "#/$defs/securityGroups" - }, "nodeDrainTimeout": { "$ref": "#/$defs/nodeDrainTimeout" } @@ -194,6 +191,9 @@ ] } }, + "additionalComputePlaneSecurityGroups": { + "$ref": "#/$defs/securityGroups" + }, "bastion": { "type": "object", "properties": { @@ -246,7 +246,7 @@ "imageNameTemplate": { "type": "object", "properties": { - "workers": { + "computePlane": { "type": "string" }, "controlPlane": { @@ -254,10 +254,10 @@ } }, "required": [ - "workers", + "computePlane", "controlPlane" ], - "additionalItems": false + "additionalProperties": false }, "common": { "type": "object", @@ -268,7 +268,7 @@ "metadata", "version", "controlPlane", - "workers" + "nodePools" ], "additionalProperties": false, "$defs": { @@ -277,7 +277,6 @@ "items": { "type": "string" }, - "minItems": 1, "uniqueItems": true }, "nodeDrainTimeout": { diff --git a/charts/t8s-cluster/values.yaml b/charts/t8s-cluster/values.yaml index feadad7126..3529494caa 100644 --- a/charts/t8s-cluster/values.yaml +++ b/charts/t8s-cluster/values.yaml @@ -10,7 +10,7 @@ global: url: https://helm.ngc.nvidia.com/nvidia charts: gpu-operator: 23.x.x - condition: '{{ eq (include "t8s-cluster.hasGPUNodes" .) "true" }}' + condition: '{{ eq (include "t8s-cluster.hasGPUNodes" (dict "context" $)) "true" }}' cloud-provider-openstack: url: https://kubernetes.github.io/cloud-provider-openstack cetic: @@ -53,7 +53,8 @@ version: minor: 25 patch: 0 -workers: {} +nodePools: {} +additionalComputePlaneSecurityGroups: [] bastion: enabled: false @@ -69,5 +70,5 @@ sshKeyName: null cni: cilium imageNameTemplate: - workers: t8s-engine-2004-kube-{{ .builtin.machineDeployment.version }} + computePlane: t8s-engine-2004-kube-{{ .builtin.machineDeployment.version }} controlPlane: t8s-engine-2004-kube-{{ .builtin.controlPlane.version }}