From 20d030d96f766a5fe4f5ac91652059c4ca83a9f9 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Mon, 25 Nov 2024 13:53:49 -0700 Subject: [PATCH 1/4] Enable ISO patching in Smee: This will allow Layer3 provisioning to be used. Signed-off-by: Jacob Weinstock --- capt/tasks/Taskfile-create.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capt/tasks/Taskfile-create.yaml b/capt/tasks/Taskfile-create.yaml index ccaeb48..b160f0e 100644 --- a/capt/tasks/Taskfile-create.yaml +++ b/capt/tasks/Taskfile-create.yaml @@ -114,7 +114,7 @@ tasks: sh: yq eval '.chart.location' {{.STATE_FILE_FQ_PATH}} CHART_NAME: tink-stack cmds: - - KUBECONFIG="{{.KUBECONFIG}}" helm install {{.CHART_NAME}} {{.LOCATION}} --version "{{.STACK_CHART_VERSION}}" --create-namespace --namespace {{.NAMESPACE}} --wait --set "global.trustedProxies={"{{.TRUSTED_PROXIES}}"}" --set "global.publicIP={{.LB_IP}}" + - KUBECONFIG="{{.KUBECONFIG}}" helm install {{.CHART_NAME}} {{.LOCATION}} --version "{{.STACK_CHART_VERSION}}" --create-namespace --namespace {{.NAMESPACE}} --wait --set "global.trustedProxies={"{{.TRUSTED_PROXIES}}"}" --set "global.publicIP={{.LB_IP}}" --set "stack.hook.extensio=both" --set "smee.iso.enabled=true" --set "smee.iso.url=http://{{.LB_IP}}:8080/hook-latest-lts-x86_64-efi-initrd.iso" --set "smee.iso.staticIPAMEnabled=true" status: - KUBECONFIG="{{.KUBECONFIG}}" helm list -n {{.NAMESPACE}} | grep -q {{.CHART_NAME}} From cd2b0a0065a19e2f5884187936d2978bd422363f Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Tue, 26 Nov 2024 20:13:23 -0700 Subject: [PATCH 2/4] Add ISO mounting capabilities: In config.yaml `bootMode` can be set to either `iso` or `netboot`. This allows for testing the all layer 3 provisioning with `iso`. Signed-off-by: Jacob Weinstock --- capt/config.yaml | 1 + capt/scripts/generate_state.sh | 1 + capt/tasks/Taskfile-capi.yaml | 6 +- capt/tasks/Taskfile-create.yaml | 11 +- capt/templates/kustomization-iso.tmpl | 300 ++++++++++++++++++ ...zation.tmpl => kustomization-netboot.tmpl} | 0 6 files changed, 317 insertions(+), 2 deletions(-) create mode 100644 capt/templates/kustomization-iso.tmpl rename capt/templates/{kustomization.tmpl => kustomization-netboot.tmpl} (100%) diff --git a/capt/config.yaml b/capt/config.yaml index 03e061c..8f938c9 100644 --- a/capt/config.yaml +++ b/capt/config.yaml @@ -33,3 +33,4 @@ virtualBMC: image: ghcr.io/jacobweinstock/virtualbmc:latest user: "root" pass: "calvin" +bootMode: netboot diff --git a/capt/scripts/generate_state.sh b/capt/scripts/generate_state.sh index 877cc35..ee0042e 100755 --- a/capt/scripts/generate_state.sh +++ b/capt/scripts/generate_state.sh @@ -67,6 +67,7 @@ cluster: controlPlane: vip: 172.18.10.75 podCIDR: 172.100.0.0/16 +bootMode: netboot EOF set -euo pipefail diff --git a/capt/tasks/Taskfile-capi.yaml b/capt/tasks/Taskfile-capi.yaml index 3614727..20d55e7 100644 --- a/capt/tasks/Taskfile-capi.yaml +++ b/capt/tasks/Taskfile-capi.yaml @@ -128,12 +128,16 @@ tasks: sh: yq eval '.versions.kube' {{.STATE_FILE_FQ_PATH}} OUTPUT_DIR: sh: yq eval '.outputDir' config.yaml + BOOTMODE: + sh: yq eval '.bootMode' {{.STATE_FILE_FQ_PATH}} + KUSTOMIZE_FILE: + sh: "[[ {{.BOOTMODE}} == 'iso' ]] && echo kustomization-iso.tmpl || echo kustomization-netboot.tmpl" sources: - config.yaml generates: - "{{.OUTPUT_DIR}}/kustomization.yaml" cmds: - - envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" < templates/kustomization.tmpl > {{.OUTPUT_DIR}}/kustomization.yaml + - envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" < templates/{{.KUSTOMIZE_FILE}} > {{.OUTPUT_DIR}}/kustomization.yaml apply-kustomization: run: once diff --git a/capt/tasks/Taskfile-create.yaml b/capt/tasks/Taskfile-create.yaml index b160f0e..ab88527 100644 --- a/capt/tasks/Taskfile-create.yaml +++ b/capt/tasks/Taskfile-create.yaml @@ -113,8 +113,17 @@ tasks: LOCATION: sh: yq eval '.chart.location' {{.STATE_FILE_FQ_PATH}} CHART_NAME: tink-stack + BOOTMODE: + sh: yq eval '.bootMode' {{.STATE_FILE_FQ_PATH}} + SET_VARS: + - global.trustedProxies={"{{.TRUSTED_PROXIES}}"} + - global.publicIP={{.LB_IP}} + - stack.hook.extension=both + - smee.iso.enabled=true + - smee.iso.url=http://{{.LB_IP}}:8080/hook-latest-lts-x86_64-efi-initrd.iso + - smee.iso.staticIPAMEnabled=true cmds: - - KUBECONFIG="{{.KUBECONFIG}}" helm install {{.CHART_NAME}} {{.LOCATION}} --version "{{.STACK_CHART_VERSION}}" --create-namespace --namespace {{.NAMESPACE}} --wait --set "global.trustedProxies={"{{.TRUSTED_PROXIES}}"}" --set "global.publicIP={{.LB_IP}}" --set "stack.hook.extensio=both" --set "smee.iso.enabled=true" --set "smee.iso.url=http://{{.LB_IP}}:8080/hook-latest-lts-x86_64-efi-initrd.iso" --set "smee.iso.staticIPAMEnabled=true" + - KUBECONFIG="{{.KUBECONFIG}}" helm install {{.CHART_NAME}} {{.LOCATION}} --version "{{.STACK_CHART_VERSION}}" --create-namespace --namespace {{.NAMESPACE}} --wait {{range .SET_VARS}}--set "{{.}}" {{end}} {{- if eq .BOOTMODE "iso" }} --set "smee.dhcp.enabled=false" --set "stack.relay.enabled=false" {{end}} status: - KUBECONFIG="{{.KUBECONFIG}}" helm list -n {{.NAMESPACE}} | grep -q {{.CHART_NAME}} diff --git a/capt/templates/kustomization-iso.tmpl b/capt/templates/kustomization-iso.tmpl new file mode 100644 index 0000000..8e1b7f4 --- /dev/null +++ b/capt/templates/kustomization-iso.tmpl @@ -0,0 +1,300 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: $NAMESPACE +resources: + - prekustomization.yaml +patches: + - target: + group: infrastructure.cluster.x-k8s.io + kind: TinkerbellMachineTemplate + name: ".*control-plane.*" + version: v1beta1 + patch: |- + - op: add + path: /spec/template/spec + value: + bootOptions: + bootMode: iso + isoURL: "http://$TINKERBELL_VIP:7171/iso/:macAddress/hook.iso" + hardwareAffinity: + required: + - labelSelector: + matchLabels: + tinkerbell.org/role: control-plane + templateOverride: | + version: "0.1" + name: playground-template + global_timeout: 6000 + tasks: + - name: "playground-template" + worker: "{{.device_1}}" + volumes: + - /dev:/dev + - /dev/console:/dev/console + - /lib/firmware:/lib/firmware:ro + actions: + - name: "stream image" + image: quay.io/tinkerbell/actions/oci2disk + timeout: 1200 + environment: + IMG_URL: $OS_REGISTRY/$OS_DISTRO-$OS_VERSION:$KUBE_VERSION.gz + DEST_DISK: {{ index .Hardware.Disks 0 }} + COMPRESSED: true + - name: "add tink cloud-init config" + image: quay.io/tinkerbell/actions/writefile + timeout: 90 + environment: + DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} + FS_TYPE: ext4 + DEST_PATH: /etc/cloud/cloud.cfg.d/10_tinkerbell.cfg + UID: 0 + GID: 0 + MODE: 0600 + DIRMODE: 0700 + CONTENTS: | + datasource: + Ec2: + metadata_urls: ["http://$TINKERBELL_VIP:50061"] + strict_id: false + system_info: + default_user: + name: tink + groups: [wheel, adm] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/bash + manage_etc_hosts: localhost + warnings: + dsid_missing_source: off + - name: "disable cloud-init networking" + image: quay.io/tinkerbell/actions/writefile + timeout: 90 + environment: + CONTENTS: 'network: {config: disabled}' + DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' + DEST_PATH: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg + DIRMODE: "0700" + FS_TYPE: ext4 + GID: "0" + MODE: "0600" + UID: "0" + - name: "create static netplan" + image: quay.io/tinkerbell/actions/writefile + timeout: 90 + environment: + CONTENTS: | + network: + version: 2 + renderer: networkd + ethernets: + id0: + match: + macaddress: {{ (index .Hardware.Interfaces 0).DHCP.MAC }} + addresses: + - {{ (index .Hardware.Interfaces 0).DHCP.IP.Address }}/16 + nameservers: + addresses: [{{ (index .Hardware.Interfaces 0).DHCP.NameServers | join ","}}] + routes: + - to: default + via: {{ (index .Hardware.Interfaces 0).DHCP.IP.Gateway }} + DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' + DEST_PATH: /etc/netplan/config.yaml + DIRMODE: "0755" + FS_TYPE: ext4 + GID: "0" + MODE: "0600" + UID: "0" + - name: "add tink cloud-init ds-config" + image: quay.io/tinkerbell/actions/writefile + timeout: 90 + environment: + DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} + FS_TYPE: ext4 + DEST_PATH: /etc/cloud/ds-identify.cfg + UID: 0 + GID: 0 + MODE: 0600 + DIRMODE: 0700 + CONTENTS: | + datasource: Ec2 + - name: "kexec image" + image: ghcr.io/jacobweinstock/waitdaemon:0.2.1 + timeout: 90 + pid: host + environment: + BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} + FS_TYPE: ext4 + IMAGE: quay.io/tinkerbell/actions/kexec + WAIT_SECONDS: 5 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - target: + group: infrastructure.cluster.x-k8s.io + kind: TinkerbellMachineTemplate + name: ".*worker.*" + version: v1beta1 + patch: |- + - op: add + path: /spec/template/spec + value: + bootOptions: + bootMode: iso + isoURL: "http://$TINKERBELL_VIP:7171/iso/:macAddress/hook.iso" + hardwareAffinity: + required: + - labelSelector: + matchLabels: + tinkerbell.org/role: worker + templateOverride: | + version: "0.1" + name: playground-template + global_timeout: 6000 + tasks: + - name: "playground-template" + worker: "{{.device_1}}" + volumes: + - /dev:/dev + - /dev/console:/dev/console + - /lib/firmware:/lib/firmware:ro + actions: + - name: "stream image" + image: quay.io/tinkerbell/actions/oci2disk + timeout: 1200 + environment: + IMG_URL: $OS_REGISTRY/$OS_DISTRO-$OS_VERSION:$KUBE_VERSION.gz + DEST_DISK: {{ index .Hardware.Disks 0 }} + COMPRESSED: true + - name: "add tink cloud-init config" + image: quay.io/tinkerbell/actions/writefile + timeout: 90 + environment: + DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} + FS_TYPE: ext4 + DEST_PATH: /etc/cloud/cloud.cfg.d/10_tinkerbell.cfg + UID: 0 + GID: 0 + MODE: 0600 + DIRMODE: 0700 + CONTENTS: | + datasource: + Ec2: + metadata_urls: ["http://$TINKERBELL_VIP:50061"] + strict_id: false + system_info: + default_user: + name: tink + groups: [wheel, adm] + sudo: ["ALL=(ALL) NOPASSWD:ALL"] + shell: /bin/bash + manage_etc_hosts: localhost + warnings: + dsid_missing_source: off + - name: "disable cloud-init networking" + image: quay.io/tinkerbell/actions/writefile + timeout: 90 + environment: + CONTENTS: 'network: {config: disabled}' + DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' + DEST_PATH: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg + DIRMODE: "0700" + FS_TYPE: ext4 + GID: "0" + MODE: "0600" + UID: "0" + - name: "create static netplan" + image: quay.io/tinkerbell/actions/writefile + timeout: 90 + environment: + CONTENTS: | + network: + version: 2 + renderer: networkd + ethernets: + id0: + match: + macaddress: {{ (index .Hardware.Interfaces 0).DHCP.MAC }} + addresses: + - {{ (index .Hardware.Interfaces 0).DHCP.IP.Address }}/16 + nameservers: + addresses: [{{ (index .Hardware.Interfaces 0).DHCP.NameServers | join ","}}] + routes: + - to: default + via: {{ (index .Hardware.Interfaces 0).DHCP.IP.Gateway }} + DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' + DEST_PATH: /etc/netplan/config.yaml + DIRMODE: "0755" + FS_TYPE: ext4 + GID: "0" + MODE: "0600" + UID: "0" + - name: "add tink cloud-init ds-config" + image: quay.io/tinkerbell/actions/writefile + timeout: 90 + environment: + DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} + FS_TYPE: ext4 + DEST_PATH: /etc/cloud/ds-identify.cfg + UID: 0 + GID: 0 + MODE: 0600 + DIRMODE: 0700 + CONTENTS: | + datasource: Ec2 + - name: "kexec image" + image: ghcr.io/jacobweinstock/waitdaemon:0.2.1 + timeout: 90 + pid: host + environment: + BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} + FS_TYPE: ext4 + IMAGE: quay.io/tinkerbell/actions/kexec + WAIT_SECONDS: 5 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - target: + group: infrastructure.cluster.x-k8s.io + kind: TinkerbellCluster + name: ".*" + version: v1beta1 + patch: |- + - op: add + path: /spec + value: + imageLookupBaseRegistry: "$OS_REGISTRY" + imageLookupOSDistro: "$OS_DISTRO" + imageLookupOSVersion: "$VERSIONS_OS" + - target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: "$CLUSTER_NAME-.*" + version: v1beta1 + patch: |- + - op: add + path: /spec/template/spec/users + value: + - name: tink + sudo: ALL=(ALL) NOPASSWD:ALL + sshAuthorizedKeys: + - $SSH_AUTH_KEY + - target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: "$CLUSTER_NAME-.*" + version: v1beta1 + patch: |- + - op: add + path: /spec/kubeadmConfigSpec/users + value: + - name: tink + sudo: ALL=(ALL) NOPASSWD:ALL + sshAuthorizedKeys: + - $SSH_AUTH_KEY + - target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: "$CLUSTER_NAME-.*" + version: v1beta1 + patch: |- + - op: add + path: /spec/kubeadmConfigSpec/preKubeadmCommands + value: + - mkdir -p /etc/kubernetes/manifests && ctr images pull ghcr.io/kube-vip/kube-vip:v$KUBEVIP_VERSION && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v$KUBEVIP_VERSION vip /kube-vip manifest pod --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address $CONTROL_PLANE_VIP --controlplane --leaderElection --k8sConfigPath $CONF_PATH > /etc/kubernetes/manifests/kube-vip.yaml diff --git a/capt/templates/kustomization.tmpl b/capt/templates/kustomization-netboot.tmpl similarity index 100% rename from capt/templates/kustomization.tmpl rename to capt/templates/kustomization-netboot.tmpl From 29b51d17873211859f82558136b0455627d30d91 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Wed, 27 Nov 2024 11:26:33 -0700 Subject: [PATCH 3/4] Bump Chart and CAPT version: Chart has Smee fixes for ISO mounting with sushy-tools. CAPT has fixes for setting the ISO url in the CAPT TinkerbellMachineTemplate spec. Signed-off-by: Jacob Weinstock --- capt/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/capt/config.yaml b/capt/config.yaml index 8f938c9..f8f131b 100644 --- a/capt/config.yaml +++ b/capt/config.yaml @@ -7,8 +7,8 @@ counts: workers: 1 spares: 1 versions: - capt: v0.6.0 - chart: 0.6.0 + capt: v0.6.1 + chart: 0.6.1 kube: v1.28.9 os: 20.04 kubevip: 0.8.7 From f9b97b1a6923f528a029f328b57b7b8750ac77b0 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Wed, 27 Nov 2024 11:28:43 -0700 Subject: [PATCH 4/4] Deploy the Helm chart based on bootMode: This enables/disables options and services depending on the boot mode. Signed-off-by: Jacob Weinstock --- capt/tasks/Taskfile-create.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/capt/tasks/Taskfile-create.yaml b/capt/tasks/Taskfile-create.yaml index ab88527..032be51 100644 --- a/capt/tasks/Taskfile-create.yaml +++ b/capt/tasks/Taskfile-create.yaml @@ -115,15 +115,18 @@ tasks: CHART_NAME: tink-stack BOOTMODE: sh: yq eval '.bootMode' {{.STATE_FILE_FQ_PATH}} - SET_VARS: + GLOBAL_VARS: - global.trustedProxies={"{{.TRUSTED_PROXIES}}"} - global.publicIP={{.LB_IP}} + ISO_VARS: - stack.hook.extension=both - smee.iso.enabled=true - smee.iso.url=http://{{.LB_IP}}:8080/hook-latest-lts-x86_64-efi-initrd.iso - smee.iso.staticIPAMEnabled=true + - smee.dhcp.enabled=false + - stack.relay.enabled=false cmds: - - KUBECONFIG="{{.KUBECONFIG}}" helm install {{.CHART_NAME}} {{.LOCATION}} --version "{{.STACK_CHART_VERSION}}" --create-namespace --namespace {{.NAMESPACE}} --wait {{range .SET_VARS}}--set "{{.}}" {{end}} {{- if eq .BOOTMODE "iso" }} --set "smee.dhcp.enabled=false" --set "stack.relay.enabled=false" {{end}} + - KUBECONFIG="{{.KUBECONFIG}}" helm install {{.CHART_NAME}} {{.LOCATION}} --version "{{.STACK_CHART_VERSION}}" --create-namespace --namespace {{.NAMESPACE}} --wait {{range .GLOBAL_VARS}}--set "{{.}}" {{end}} {{- if eq .BOOTMODE "iso" }} {{- range .ISO_VARS }}--set "{{.}}" {{end}} {{end}} status: - KUBECONFIG="{{.KUBECONFIG}}" helm list -n {{.NAMESPACE}} | grep -q {{.CHART_NAME}}