ci: update base os image during update dependencies workflow (#349)

Issue #, if available: Base OS image updates have been manual by maintainers. *Description of changes:* This change adds update base os image to the update dependencies workflow. *Testing done:* Locally tested regex to validate it produced the same result as the last update: ``` macedonv@localhost:/Volumes/workplace/finch-core$ aws s3 ls s3://*** --recursive | grep -E 'Fedora-Cloud-Base-.*\.aarch64-[0-9]+\.qcow2$$' | tail -n 1 | sort | awk '{print $4}' Fedora-Cloud-Base-40-1.14.aarch64-20240626221713.qcow2 macedonv@localhost:/Volumes/workplace/finch-core$ ``` - [x] I've reviewed the guidance in CONTRIBUTING.md #### License Acceptance By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. --------- Signed-off-by: Austin Vazquez <[email protected]>
runfinch · Jul 22, 2024 · c059f8d · c059f8d
1 parent f62e43b
commit c059f8d
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 5 deletions.
diff --git a/.github/workflows/update-dependencies.yaml b/.github/workflows/update-dependencies.yaml
@@ -25,11 +25,14 @@ jobs:
           role-session-name: dependency-upload-session
           aws-region: ${{ secrets.REGION }}
 
-      # This step fetches the latest set of released dependencies from s3 and updates the Makefile to use the same.
-      - name: update dependencies url
-        run: |
-          ./bin/update-deps.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
-          ./bin/update-rootfs.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
+      - name: Update Lima dependencies archive for Finch on macOS
+        run: bash bin/update-deps.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
+
+      - name: Update base OS for Finch on macOS
+        run: bash bin/update-os-image.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
+
+      - name: Update rootfs for Finch on Windows
+        run: bash bin/update-rootfs.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
 
       - name: create PR
         uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0

diff --git a/bin/update-deps.sh b/bin/update-deps.sh
@@ -37,12 +37,16 @@ aarch64_deps=$(find_latest_object_match_from_s3 "${AARCH64_FILENAME_PATTERN}" "$
 aarch64_deps_shasum_url="${DEPENDENCY_CLOUDFRONT_URL}/${aarch64_deps}.sha512sum"
 aarch64_deps_shasum=$(curl -L --fail "${aarch64_deps_shasum_url}")
 
+pull_artifact_and_verify_shasum "${DEPENDENCY_CLOUDFRONT_URL}/${aarch64_deps}" "${aarch64_deps_shasum}"
+
 amd64_deps=$(find_latest_object_match_from_s3 "${AMD64_FILENAME_PATTERN}" "${dependency_bucket}/${X86_64}")
 [[ -z "$amd64_deps" ]] && { echo "Error: x86_64 dependency not found"; exit 1; }
 
 amd64_deps_shasum_url="${DEPENDENCY_CLOUDFRONT_URL}/${amd64_deps}.sha512sum"
 amd64_deps_shasum=$(curl -L --fail "${amd64_deps_shasum_url}")
 
+pull_artifact_and_verify_shasum "${DEPENDENCY_CLOUDFRONT_URL}/${amd64_deps}" "${amd64_deps_shasum}"
+
 # Update bundles file with latest artifacts and digests.
 BUNDLES_FILE="${PROJECT_ROOT}/deps/lima-bundles.conf"
 truncate -s 0 "${BUNDLES_FILE}"

diff --git a/bin/update-os-image.sh b/bin/update-os-image.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# A script to update the base os image used for Finch on macOS.
+#
+# Usage: bash update-os-image.sh -d <S3 bucket>
+
+set -euxo pipefail
+
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd -- "${CURRENT_DIR}/.." && pwd)"
+
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/bin/utility.sh"
+
+DEPENDENCY_CLOUDFRONT_URL="https://deps.runfinch.com"
+AARCH64_FILENAME_PATTERN="Fedora-Cloud-Base-.*\.aarch64-[0-9]+\.qcow2$"
+AMD64_FILENAME_PATTERN="Fedora-Cloud-Base-.*\.x86_64-[0-9]+\.qcow2$"
+
+while getopts d: flag
+do
+  case "${flag}" in
+    d) dependency_bucket=${OPTARG};;
+    *) echo "Error: unknown flag" && exit 1;;
+  esac
+done
+
+[[ -z "$dependency_bucket" ]] && { echo "Error: dependency bucket not set"; exit 1; }
+
+aarch64_deps=$(find_latest_object_match_from_s3 "${AARCH64_FILENAME_PATTERN}" "${dependency_bucket}")
+[[ -z "$aarch64_deps" ]] && { echo "Error: aarch64 dependency not found"; exit 1; }
+
+# Need to pull the shasum of the artifact to store for later verification.
+aarch64_deps_shasum_url="${DEPENDENCY_CLOUDFRONT_URL}/${aarch64_deps}.sha512sum"
+aarch64_deps_shasum=$(curl -L --fail "${aarch64_deps_shasum_url}")
+
+pull_artifact_and_verify_shasum "${DEPENDENCY_CLOUDFRONT_URL}/${aarch64_deps}" "${aarch64_deps_shasum}"
+
+amd64_deps=$(find_latest_object_match_from_s3 "${AMD64_FILENAME_PATTERN}" "${dependency_bucket}")
+[[ -z "$amd64_deps" ]] && { echo "Error: x86_64 dependency not found"; exit 1; }
+
+amd64_deps_shasum_url="${DEPENDENCY_CLOUDFRONT_URL}/${amd64_deps}.sha512sum"
+amd64_deps_shasum=$(curl -L --fail "${amd64_deps_shasum_url}")
+
+pull_artifact_and_verify_shasum "${DEPENDENCY_CLOUDFRONT_URL}/${amd64_deps}" "${amd64_deps_shasum}"
+
+# Update base os file with latest artifacts and digests
+OS_FILE="${PROJECT_ROOT}/deps/full-os.conf"
+truncate -s 0 "${OS_FILE}"
+{
+    echo "ARTIFACT_BASE_URL=${DEPENDENCY_CLOUDFRONT_URL}"
+    echo ""
+    echo "# From https://dl.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/aarch64/images/"
+    echo "AARCH64_ARTIFACT=$(basename "${aarch64_deps}")"
+    echo "AARCH64_512_DIGEST=${aarch64_deps_shasum}"
+    echo ""
+    echo "# From https://dl.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/"
+    echo "X86_64_ARTIFACT=$(basename "${amd64_deps}")"
+    echo "X86_64_512_DIGEST=${amd64_deps_shasum}"
+} >> "${OS_FILE}"
diff --git a/bin/update-rootfs.sh b/bin/update-rootfs.sh
@@ -38,6 +38,8 @@ amd64_deps=$(find_latest_object_match_from_s3 "${AMD64_FILENAME_PATTERN}" "${dep
 amd64_deps_shasum_url="${DEPENDENCY_CLOUDFRONT_URL}/${amd64_deps}.sha512sum"
 amd64_deps_shasum=$(curl -L --fail "${amd64_deps_shasum_url}")
 
+pull_artifact_and_verify_shasum "${DEPENDENCY_CLOUDFRONT_URL}/${amd64_deps}" "${amd64_deps_shasum}"
+
 # Update rootfs file with latest artifacts and digests
 ROOTFS_FILE="${PROJECT_ROOT}/deps/rootfs.conf"
 truncate -s 0 "${ROOTFS_FILE}"

diff --git a/bin/utility.sh b/bin/utility.sh
@@ -22,3 +22,20 @@ find_latest_object_match_from_s3() {
 
     echo "$object"
 }
+
+# pull_artifact_and_verify_shasum is a function for pulling a Finch core
+# artifact and verifying its shasum.
+#
+# @param artifact_url - URL to artifact
+# @param expected_shasum - the expected SHA512SUM for the artifact
+pull_artifact_and_verify_shasum() {
+    local artifact_url="$1"
+    local expected_shasum="$2"
+
+    local artifact
+    artifact=$(basename "$artifact_url")
+
+    curl -L --fail "${artifact_url}" > "${artifact}"
+    shasum --algorithm 512 "${artifact}" | cut -d ' ' -f 1 | grep -xq "^${expected_shasum}$" || \
+      (echo "error: shasum verification failed for \"${artifact}\" dependency" && rm -f "${artifact}" && exit 1)
+}