From fb8dd8a61dda1818e89fa32ab3037ed15b2a9dfa Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Tue, 12 Nov 2024 11:59:19 +0000 Subject: [PATCH 1/7] ENH: Only config locales we need Instead of configuring en-ZW for example, replace the default locale with en_US which is shipped anyway since these VMs are not user facing in any-way --- amphora-image-builder/Dockerfile | 1 - amphora-image-builder/build.sh | 2 +- .../elements/vm_baseline/install.d/90-run-vm_baseline | 11 ++++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/amphora-image-builder/Dockerfile b/amphora-image-builder/Dockerfile index 7c103c0..02309ef 100644 --- a/amphora-image-builder/Dockerfile +++ b/amphora-image-builder/Dockerfile @@ -10,7 +10,6 @@ RUN apt-get update && \ # rather than main COPY ./ /opt/cloud-image-builders - COPY amphora-image-builder/entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/amphora-image-builder/build.sh b/amphora-image-builder/build.sh index 31a9cb5..d405525 100755 --- a/amphora-image-builder/build.sh +++ b/amphora-image-builder/build.sh @@ -9,5 +9,5 @@ docker build -f amphora-image-builder/Dockerfile -t amphora-image-builder:local # The Amphora builder requires privileged access to the host # to mount /proc and /sys -docker run --privileged -v "$(pwd)":/output amphora-image-builder:local +docker run --privileged -v "$(pwd)/output":/output amphora-image-builder:local cd amphora-image-builder diff --git a/amphora-image-builder/elements/vm_baseline/install.d/90-run-vm_baseline b/amphora-image-builder/elements/vm_baseline/install.d/90-run-vm_baseline index 829eb56..4ca80d2 100644 --- a/amphora-image-builder/elements/vm_baseline/install.d/90-run-vm_baseline +++ b/amphora-image-builder/elements/vm_baseline/install.d/90-run-vm_baseline @@ -11,12 +11,11 @@ source /tmp/amphora-venv/bin/activate /usr/bin/git clone https://github.com/stfc/cloud-image-builders.git cd cloud-image-builders -apt-get install language-pack-en -y -locale-gen en_GB.UTF-8 -dpkg-reconfigure --frontend noninteractive locales -update-locale LC_ALL=en_GB.UTF-8 LANG=en_GB.UTF-8 +apt-get install locale -y && apt-get clean all +echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ +echo "LANG=en_US.UTF-8" > /etc/locale.conf && \ +locale-gen en_US.UTF-8 source /etc/default/locale -locale apt-get install gpg-agent -y /usr/bin/gpg-agent --daemon @@ -26,3 +25,5 @@ sed -i "s/hosts: default/hosts: localhost/g" os_builders/*.yml mkdir -p /var/ossec/etc/extra echo "{\"groups\": [\"default\", \"cloud\", \"ubuntu\", \"debian\", \"octavia-amphora\"], \"labels\": {\"amphora-build-date\": \"$(date '+%Y-%m-%d %H:%M:%S')\" }}" > /var/ossec/etc/extra/03-amphora.json ansible-playbook os_builders/prepare_user_image.yml --extra-vars provision_this_machine=true -i os_builders/inventory/localhost.yml + +apt-get autoremove -y From ce83b7ff46d6dedb08c7d03faa69cadb7cb57c55 Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Tue, 12 Nov 2024 17:40:20 +0000 Subject: [PATCH 2/7] ENH: Add output related files to gitignore Adds the output directory and any log.txt files from the build to the gitignore to ensure we don't pollute the git history with large or obsolete files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f3ae762..e47ad1b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Amphora output files amphora-x64*.raw amphora-x64*.d/ +log.txt # Byte-compiled / optimized / DLL files __pycache__/ @@ -22,6 +23,7 @@ lib/ lib64/ parts/ sdist/ +output/ var/ wheels/ share/python-wheels/ From 5a1363ba5fcb8ece653ba84a4e020c6a51ec7265 Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Tue, 12 Nov 2024 17:41:34 +0000 Subject: [PATCH 3/7] ENH: Switch builder to use 22.04 Switches the builder to use Ubuntu 22.04 Unfortunately, upstream disk image builder requires a fix as they use pkg_resources for 24.04 support, so bump a single version instead --- amphora-image-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amphora-image-builder/Dockerfile b/amphora-image-builder/Dockerfile index 02309ef..10fcde8 100644 --- a/amphora-image-builder/Dockerfile +++ b/amphora-image-builder/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 RUN apt-get update && \ apt-get install -y --no-install-recommends \ From 0b8bc08b51e8e7f54167ea391485f47c7bf5c7b8 Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Tue, 12 Nov 2024 17:42:29 +0000 Subject: [PATCH 4/7] MAINT: Improve git clone for amphora Improves the various git clones for the amphora: - Use OpenDev directly for Octavia instead of a mirror - Set depth to 1 for all clones, since these are ephemeral --- .../elements/vm_baseline/install.d/90-run-vm_baseline | 2 +- amphora-image-builder/entrypoint.sh | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/amphora-image-builder/elements/vm_baseline/install.d/90-run-vm_baseline b/amphora-image-builder/elements/vm_baseline/install.d/90-run-vm_baseline index 4ca80d2..98b6a67 100644 --- a/amphora-image-builder/elements/vm_baseline/install.d/90-run-vm_baseline +++ b/amphora-image-builder/elements/vm_baseline/install.d/90-run-vm_baseline @@ -8,7 +8,7 @@ apt-get install python3-pip python3-venv -y /usr/bin/python3 -m venv /tmp/amphora-venv source /tmp/amphora-venv/bin/activate /usr/bin/python3 -m pip install ansible -/usr/bin/git clone https://github.com/stfc/cloud-image-builders.git +/usr/bin/git clone https://github.com/stfc/cloud-image-builders.git --depth=1 cd cloud-image-builders apt-get install locale -y && apt-get clean all diff --git a/amphora-image-builder/entrypoint.sh b/amphora-image-builder/entrypoint.sh index 404e9a1..88ffef8 100755 --- a/amphora-image-builder/entrypoint.sh +++ b/amphora-image-builder/entrypoint.sh @@ -2,12 +2,15 @@ set -euxo pipefail +DIB_RELEASE="jammy" # Ubuntu 22.04 +export DIB_RELEASE + cd /tmp python3 -m virtualenv octavia_disk_image_create # shellcheck source=/dev/null source octavia_disk_image_create/bin/activate -git clone --depth=1 https://github.com/openstack/octavia +git clone --depth=1 https://opendev.org/openstack/octavia DIB_REPO_PATH="$(pwd)/octavia" export DIB_REPO_PATH From 4aef55ff108ab9cca1cc9ddd068661c172b1ded5 Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Tue, 12 Nov 2024 17:47:45 +0000 Subject: [PATCH 5/7] DOC: Update README to use output dir Update the upload step to use the dedicated output directory, instead of making the user manually cd to the correct place --- amphora-image-builder/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amphora-image-builder/README b/amphora-image-builder/README index 9bc175d..9400f66 100644 --- a/amphora-image-builder/README +++ b/amphora-image-builder/README @@ -20,5 +20,5 @@ Usage # Set as appropriate to your environment export TARGET_PROJECT=service -openstack image create --property hw_disk_bus=virtio --property hw_scsi_model=virtio-scsi --property hw_vif_multiqueue_enabled=true --private --project $TARGET_PROJECT --disk-format qcow2 --container-format bare --file amphora-x64-haproxy.qcow2 --progress amphora-x64-$(date +%Y-%m-%d)-haproxy +openstack image create --property hw_disk_bus=virtio --property hw_scsi_model=virtio-scsi --property hw_vif_multiqueue_enabled=true --private --project $TARGET_PROJECT --disk-format qcow2 --container-format bare --file ../output/amphora-x64-$(date +%Y-%m-%d)-haproxy.qcow2 --progress amphora-x64-$(date +%Y-%m-%d)-haproxy ``` From e11313ec12dd045f1c91d7f36223289a6f43e0a6 Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Tue, 12 Nov 2024 17:55:14 +0000 Subject: [PATCH 6/7] ENH: Validate and upload artifact from build Validates the qemu image then creates artifacts from the Github Actions including the SHA256 (which OpenStack can use as part of the image upload) as artifacts that stay around for ~1 week --- .github/workflows/amphora_builder.yaml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/amphora_builder.yaml b/.github/workflows/amphora_builder.yaml index a8e502e..919cf60 100644 --- a/.github/workflows/amphora_builder.yaml +++ b/.github/workflows/amphora_builder.yaml @@ -24,5 +24,29 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Exec Amphora build - run: cd amphora-image-builder && ./build.sh + run: | + cd $GITHUB_WORKSPACE/amphora-image-builder + ./build.sh + sudo chown -R $USER:$USER $GITHUB_WORKSPACE/output + + - name: Check image is valid + run: | + cd $GITHUB_WORKSPACE + sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-utils + qemu-img check output/amphora-x64-$(date +%Y-%m-%d)-haproxy.qcow2 + + - name: SHA256 image + run: | + cd $GITHUB_WORKSPACE + sha256sum output/amphora-x64-$(date +%Y-%m-%d)-haproxy.qcow2 > output/amphora-x64-$(date +%Y-%m-%d)-haproxy.qcow2.sha256 + + - name: Upload image + uses: actions/upload-artifact@v4 + with: + name: amphora-image + if-no-files-found: error + path: output/*qcow2* + compression-level: 0 # qcow2 files are already compressed + retention-days: 7 From ea9462b2ce63d782ab8c3f1c43d0c5e976ed2d9b Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Tue, 19 Nov 2024 14:46:40 +0000 Subject: [PATCH 7/7] DOC: add steps for enabling the Amphora image Adds steps for enabling the Amphora image after uploading it --- amphora-image-builder/README | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/amphora-image-builder/README b/amphora-image-builder/README index 9400f66..cdeb8dd 100644 --- a/amphora-image-builder/README +++ b/amphora-image-builder/README @@ -22,3 +22,24 @@ export TARGET_PROJECT=service openstack image create --property hw_disk_bus=virtio --property hw_scsi_model=virtio-scsi --property hw_vif_multiqueue_enabled=true --private --project $TARGET_PROJECT --disk-format qcow2 --container-format bare --file ../output/amphora-x64-$(date +%Y-%m-%d)-haproxy.qcow2 --progress amphora-x64-$(date +%Y-%m-%d)-haproxy ``` + +Enabling this for the Amphora +----------------------------- + +- Share this image with the admin project so the control plane can see it after it's tagged + +``` +openstack image set --shared --project admin amphora-x64-$(date +%Y-%m-%d)-haproxy +openstack image add project amphora-x64-$(date +%Y-%m-%d)-haproxy admin +openstack image set --accept --project admin amphora-x64-$(date +%Y-%m-%d)-haproxy +``` + +- Tag this image with the Amphora image tag +``` +openstack image set --tag amphora amphora-x64-$(date +%Y-%m-%d)-haproxy +``` + +- Untag the old image after testing +``` +openstack image unset --tag amphora amphora- +```