diff --git a/.github/bin/update-os-image.sh b/.github/bin/update-os-image.sh deleted file mode 100755 index 295d64efe..000000000 --- a/.github/bin/update-os-image.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -euxo pipefail -# Set OS hash directory in finch-core as base directory for searching latest OS images. -OS_BASE_IMAGE_HASH_DIR="./deps/finch-core/hashes/" - -OS_AARCH64_FILENAME_PATTERN="Fedora-Cloud-Base-.*-.*.aarch64-.*.qcow2" - -OS_X86_64_FILENAME_PATTERN="Fedora-Cloud-Base-.*-.*.x86_64-.*.qcow2" - - -# Use wildcard patterns to search for the two files and assign their paths to variables -AARCH64_FILEPATH=$(find "$OS_BASE_IMAGE_HASH_DIR" -name "Fedora-Cloud-Base-*-*.aarch64-*.qcow2.sha512" -print -quit) -X86_64_FILEPATH=$(find "$OS_BASE_IMAGE_HASH_DIR" -name "Fedora-Cloud-Base-*-*.x86_64-*.qcow2.sha512" -print -quit) - -# Extract the file names without the path and remove the ".sha512" extension -AARCH64_OS_BASE_IMAGE=$(basename "$AARCH64_FILEPATH" .sha512) -X86_64_OS_BASE_IMAGE=$(basename "$X86_64_FILEPATH" .sha512) - -echo "AARCH64 base image: ${AARCH64_OS_BASE_IMAGE}" -echo "X86_64 base image: ${X86_64_OS_BASE_IMAGE}" - -# Replace occurrences of FINCH_OS_BASENAME in the Makefile with the file names -sed -E -i.bak 's|^([[:blank:]]*FINCH_OS_BASENAME[[:blank:]]*\?=[[:blank:]]*)('"${OS_AARCH64_FILENAME_PATTERN}"')|\1'"$AARCH64_OS_BASE_IMAGE"'|' Makefile -sed -E -i.bak 's|^([[:blank:]]*FINCH_OS_BASENAME[[:blank:]]*\?=[[:blank:]]*)('"${OS_X86_64_FILENAME_PATTERN}"')|\1'"$X86_64_OS_BASE_IMAGE"'|' Makefile diff --git a/.github/workflows/sync-submodules-and-deps.yaml b/.github/workflows/sync-submodules-and-deps.yaml index 1ac1cb357..a66d9e484 100644 --- a/.github/workflows/sync-submodules-and-deps.yaml +++ b/.github/workflows/sync-submodules-and-deps.yaml @@ -34,18 +34,6 @@ jobs: run: | git submodule update --remote - - name: Update OS image - run: | - ./.github/bin/update-os-image.sh - - - name: Update lima dependencies url - run: | - ./deps/finch-core/bin/update-deps.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }} - - - name: Update rootfs - run: | - ./deps/finch-core/bin/update-rootfs.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }} - - name: Create PR uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 with: @@ -55,9 +43,9 @@ jobs: body: > Bump for Finch's dependencies. - Currently, this updates values based on artifacts defined in the finch-core repository, - such as: operating system images, and the version of Lima (and it's dependencies), - which are bundled as `lima-and-qemu.macos-*.*.tar.gz` archives. + Updates finch-core submodule which defines the core dependencies for Finch on each respective + platform such as: operating system images and Lima bundle (Lima version and it's dependencies) + as `lima-and-qemu.macos-*.*.tar.gz` archives. Since updating the Lima version may also update the default version of the `nerdctl-full` archive (which contains nerdctl, containerd, buildkit, etc.), this may require a manual diff --git a/Makefile b/Makefile index 13a9fc713..b587f2572 100644 --- a/Makefile +++ b/Makefile @@ -7,12 +7,17 @@ PREFIX ?= $(CURDIR)/_output DEST := $(shell echo "$(DESTDIR)/$(PREFIX)" | sed 's:///*:/:g; s://*$$::') BINDIR ?= /usr/local/bin OUTDIR ?= $(CURDIR)/_output +OS_OUTDIR ?= $(OUTDIR)/os + +OUTPUT_DIRECTORIES := $(OUTDIR) $(OS_OUTDIR) +$(OUTPUT_DIRECTORIES): + @mkdir -p $@ + PACKAGE := github.com/runfinch/finch BINARYNAME := finch LIMA_FILENAME := lima LIMA_EXTENSION := .tar.gz -LIMA_HOME := $(DEST)/lima/data # Created by the CLI after installation, only used in uninstall step LIMA_VDE_SUDOERS_FILE := /etc/sudoers.d/finch-lima # Final installation prefix for vde created by CLI after installation, only used in uninstall step @@ -20,7 +25,6 @@ VDE_INSTALL ?= /opt/finch UNAME := $(shell uname -m) ARCH ?= $(UNAME) SUPPORTED_ARCH = false -CORE_VDE_PREFIX ?= $(OUTDIR)/dependencies/vde/opt/finch LICENSEDIR := $(OUTDIR)/license-files VERSION := $(shell git describe --match 'v[0-9]*' --dirty='.modified' --always --tags) GITCOMMIT := $(shell git rev-parse HEAD)$(shell test -z "$(git status --porcelain)" || echo .m) @@ -30,9 +34,6 @@ MIN_MACOS_VERSION ?= 11.0 GOOS ?= $(shell $(GO) env GOOS) ifeq ($(GOOS),windows) BINARYNAME := $(addsuffix .exe, $(BINARYNAME)) -sha = sha256sum -else -sha = shasum -a 256 endif .DEFAULT_GOAL := all @@ -42,129 +43,31 @@ REGISTRY ?= "" ifneq (,$(findstring arm64,$(ARCH))) SUPPORTED_ARCH = true LIMA_ARCH = aarch64 - # From https://dl.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/aarch64/images/ - FINCH_OS_BASENAME ?= Fedora-Cloud-Base-40-1.14.aarch64-20240514214641.qcow2 - LIMA_URL ?= https://deps.runfinch.com/aarch64/lima-and-qemu.macos-aarch64.1715678889.tar.gz else ifneq (,$(findstring x86_64,$(ARCH))) SUPPORTED_ARCH = true LIMA_ARCH = x86_64 - # From https://dl.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/ - FINCH_OS_BASENAME ?= Fedora-Cloud-Base-40-1.14.x86_64-20240514214655.qcow2 - LIMA_URL ?= https://deps.runfinch.com/x86-64/lima-and-qemu.macos-x86_64.1715678889.tar.gz - FINCH_ROOTFS_URL ?= https://deps.runfinch.com/common/x86-64/finch-rootfs-production-amd64-1715724303.tar.gz - FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_URL)) endif -FINCH_OS_HASH := `$(sha) $(OUTDIR)/os/$(FINCH_OS_BASENAME) | cut -d ' ' -f 1` -FINCH_OS_DIGEST := "sha256:$(FINCH_OS_HASH)" -FINCH_OS_IMAGE_LOCATION_ROOT ?= $(DEST) -FINCH_OS_IMAGE_LOCATION ?= $(FINCH_OS_IMAGE_LOCATION_ROOT)/os/$(FINCH_OS_BASENAME) - -# TODO: Windows PoC extracting rootfs... -FINCH_ROOTFS_HASH := `$(sha) $(OUTDIR)/os/$(FINCH_ROOTFS_BASENAME) | cut -d ' ' -f 1` -FINCH_ROOTFS_DIGEST := "sha256:$(FINCH_ROOTFS_HASH)" -FINCH_ROOTFS_LOCATION_ROOT ?= $(DEST)/ -FINCH_ROOTFS_LOCATION ?= $(FINCH_ROOTFS_LOCATION_ROOT)os/$(FINCH_ROOTFS_BASENAME) - .PHONY: arch-test arch-test: @if [ $(SUPPORTED_ARCH) != "true" ]; then echo "Unsupported architecture: $(ARCH)"; exit "1"; fi -.PHONY: all -ifeq ($(GOOS),windows) -all: arch-test finch finch-core-local finch.windows.yaml networks.yaml config.yaml +BUILD_OS ?= $(OS) +FINCH_CORE_DIR := $(CURDIR)/deps/finch-core +ifeq ($(BUILD_OS), Windows_NT) +include Makefile.windows else -all: arch-test finch finch-core finch.yaml networks.yaml config.yaml lima-and-qemu +include Makefile.darwin endif -.PHONY: all-local -all-local: arch-test networks.yaml config.yaml lima-and-qemu local-core finch.yaml - -.PHONY: finch-core -finch-core: - cd deps/finch-core && \ - FINCH_OS_AARCH64_URL="$(FINCH_OS_AARCH64_URL)" \ - VDE_TEMP_PREFIX=$(CORE_VDE_PREFIX) \ - "$(MAKE)" - - mkdir -p _output - cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR) - rm -rf $(OUTDIR)/lima-template - -.PHONY: finch-core-local -finch-core-local: - cd deps/finch-core && \ - FINCH_OS_x86_URL="$(FINCH_OS_x86_URL)" \ - FINCH_OS_AARCH64_URL="$(FINCH_OS_AARCH64_URL)" \ - VDE_TEMP_PREFIX=$(CORE_VDE_PREFIX) \ - "$(MAKE)" all lima - - mkdir -p _output - cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR) - rm -rf $(OUTDIR)/lima-template - -.PHONY: local-core -local-core: - cd deps/finch-core && \ - FINCH_OS_x86_URL="$(FINCH_OS_x86_URL)" \ - FINCH_OS_AARCH64_URL="$(FINCH_OS_AARCH64_URL)" \ - VDE_TEMP_PREFIX=$(CORE_VDE_PREFIX) \ - "$(MAKE)" lima lima-socket-vmnet - - mkdir -p _output - cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR) - cd deps/finch-core/src/lima/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)/lima - cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR) - cd deps/finch-core/src/lima/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)/lima - rm -rf $(OUTDIR)/lima-template - -.PHONY: lima-and-qemu -lima-and-qemu: networks.yaml - mkdir -p $(OUTDIR)/downloads - # download artifacts - curl -L $(LIMA_URL) > $(OUTDIR)/downloads/lima-and-qemu.tar.gz - - # Untar LIMA - tar -xvf $(OUTDIR)/downloads/lima-and-qemu.tar.gz -C $(OUTDIR)/lima/ - - # Delete downloads - rm -rf $(OUTDIR)/downloads - - -FINCH_IMAGE_LOCATION ?= -FINCH_IMAGE_DIGEST ?= -ifeq ($(GOOS),windows) - # Because the path in windows /C:/ is not an Absolute path, prefix with file:/ which is handled by lima https://github.com/lima-vm/lima/blob/da1260dc87fb30345c3ee7bfb131c29646e26d10/pkg/downloader/downloader.go#L266 - FINCH_IMAGE_LOCATION := "file:/$(FINCH_ROOTFS_LOCATION)" - FINCH_IMAGE_DIGEST := $(FINCH_ROOTFS_DIGEST) -else - FINCH_IMAGE_LOCATION := $(FINCH_OS_IMAGE_LOCATION) - FINCH_IMAGE_DIGEST := $(FINCH_OS_DIGEST) -endif -.PHONY: finch.yaml -finch.yaml: finch-core - mkdir -p $(OUTDIR)/os - # merge the appropriate YAMLs - cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' mac.yaml common.yaml > ../finch.yaml - cp finch.yaml $(OUTDIR)/os - # using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed - # this sed command uses the alternative separator of "|" because the image location uses "/" - sed -i.bak -e "s||$(FINCH_IMAGE_LOCATION)|g" $(OUTDIR)/os/finch.yaml - sed -i.bak -e "s//$(LIMA_ARCH)/g" $(OUTDIR)/os/finch.yaml - sed -i.bak -e "s//$(FINCH_IMAGE_DIGEST)/g" $(OUTDIR)/os/finch.yaml - -# TODO: Windows PoC - clean this up / consolidate +all: arch-test finch install.finch-core-dependencies finch.yaml networks.yaml config.yaml + +.PHONY: install.finch-core-dependencies +install.finch-core-dependencies: + OUTDIR=$(OUTDIR) "$(MAKE)" -C $(FINCH_CORE_DIR) install.dependencies + .PHONY: finch.yaml -finch.windows.yaml: finch-core-local - mkdir -p $(OUTDIR)/os - # merge the appropriate YAMLs - cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' windows.yaml common.yaml > ../finch.windows.yaml - cp finch.windows.yaml $(OUTDIR)/os/finch.yaml - # using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed - # this sed command uses the alternative separator of "|" because the image location uses "/" - sed -i.bak -e "s||$(FINCH_IMAGE_LOCATION)|g" $(OUTDIR)/os/finch.yaml - sed -i.bak -e "s//$(LIMA_ARCH)/g" $(OUTDIR)/os/finch.yaml - sed -i.bak -e "s//$(FINCH_IMAGE_DIGEST)/g" $(OUTDIR)/os/finch.yaml +finch.yaml: $(OS_OUTDIR)/finch.yaml .PHONY: networks.yaml networks.yaml: @@ -402,7 +305,7 @@ mdlint-ctr: ifeq ($(GOOS),windows) clean: -@rm -rf $(OUTDIR) 2>/dev/null || true - -@rm -rf ./deps/finch-core/_output || true + -@"$(MAKE)" -C $(FINCH_CORE_DIR) clean -@rm ./*.tar.gz 2>/dev/null || true -@rm ./*.qcow2 2>/dev/null || true -@rm ./test-coverage.* 2>/dev/null || true @@ -415,8 +318,7 @@ clean: -sudo rm -rf "/private/var/run/finch-lima" -sudo rm -rf "/private/etc/sudoers.d/finch-lima" -@rm -rf $(OUTDIR) 2>/dev/null || true - -@rm -rf ./deps/finch-core/_output || true - -@rm -rf ./deps/finch-core/downloads/os/$(FINCH_OS_BASENAME) || true + -@$(MAKE) -C $(FINCH_CORE_DIR) clean -@rm ./*.tar.gz 2>/dev/null || true -@rm ./*.qcow2 2>/dev/null || true -@rm ./test-coverage.* 2>/dev/null || true diff --git a/Makefile.darwin b/Makefile.darwin new file mode 100644 index 000000000..90d0aa7c4 --- /dev/null +++ b/Makefile.darwin @@ -0,0 +1,34 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# For Finch on macOS, the base OS location and digest are set +# based on the values set in deps/finch-core/deps/full-os.conf +-include $(FINCH_CORE_DIR)/deps/full-os.conf +ifeq ($(ARCH),x86_64) +FINCH_OS_BASENAME=$(X86_64_ARTIFACT) +FINCH_OS_DIGEST=$(X86_64_512_DIGEST) +else +FINCH_OS_BASENAME=$(AARCH64_ARTIFACT) +FINCH_OS_DIGEST=$(AARCH64_512_DIGEST) +endif + +FINCH_IMAGE_LOCATION:=$(OS_OUTDIR)/$(FINCH_OS_BASENAME) +FINCH_IMAGE_DIGEST:="sha512:$(FINCH_OS_DIGEST)" + +.PHONY: finch.yaml +finch.yaml: $(OS_OUTDIR)/finch.yaml + +$(OS_OUTDIR)/finch.yaml: $(OS_OUTDIR) finch.yaml.d/common.yaml finch.yaml.d/mac.yaml + # merge the appropriate YAMLs into a temporary finch.yaml file on the current working directory + cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' mac.yaml common.yaml > ../finch.yaml.temp + + # using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed + # this sed command uses the alternative separator of "|" because the image location uses "/" + sed -i.bak -e "s||$(FINCH_IMAGE_LOCATION)|g" finch.yaml.temp + sed -i.bak -e "s//$(LIMA_ARCH)/g" finch.yaml.temp + sed -i.bak -e "s//$(FINCH_IMAGE_DIGEST)/g" finch.yaml.temp + + # Replacement was successful, so cleanup .bak + @rm finch.yaml.temp.bak + + mv finch.yaml.temp $@ diff --git a/Makefile.windows b/Makefile.windows new file mode 100644 index 000000000..797bdcbd5 --- /dev/null +++ b/Makefile.windows @@ -0,0 +1,32 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# For Finch on Windows, the rootfs location and digest are set +# based on the values in deps/finch-core/deps/rootfs.conf +-include $(FINCH_CORE_DIR)/deps/rootfs.conf +ifeq ($(ARCH),x86_64) +FINCH_ROOTFS_BASENAME ?= $(X86_64_ARTIFACT) +FINCH_ROOTFS_DIGEST ?= $(X86_64_512_DIGEST) +else +$(error Finch on Windows ARM not supported) +endif + +FINCH_IMAGE_LOCATION := $(OS_OUTDIR)/$(FINCH_ROOTFS_BASENAME) +FINCH_IMAGE_DIGEST := "sha512:$(FINCH_ROOTFS_DIGEST)" + +$(OS_OUTDIR)/finch.yaml: $(OS_OUTDIR) finch.yaml.d/common.yaml finch.yaml.d/windows.yaml + # merge the appropriate YAMLs into a temporary finch.yaml file on the current working directory + cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' windows.yaml common.yaml > ../finch.yaml.temp + + # using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed + # this sed command uses the alternative separator of "|" because the image location uses "/" + # Because the path in windows /C:/ is not an Absolute path, prefix with file:/ which is handled by lima + # https://github.com/lima-vm/lima/blob/da1260dc87fb30345c3ee7bfb131c29646e26d10/pkg/downloader/downloader.go#L266 + sed -i.bak -e "s||file://$(FINCH_IMAGE_LOCATION)|g" finch.yaml.temp + sed -i.bak -e "s//$(LIMA_ARCH)/g" finch.yaml.temp + sed -i.bak -e "s//$(FINCH_IMAGE_DIGEST)/g" finch.yaml.temp + + # Replacement was successful, so cleanup .bak + @rm finch.yaml.temp.bak + + mv finch.yaml.temp $@ diff --git a/deps/finch-core b/deps/finch-core index 5c71e7350..772bd430d 160000 --- a/deps/finch-core +++ b/deps/finch-core @@ -1 +1 @@ -Subproject commit 5c71e73504c57332b98fdd1a55dd845db70ad419 +Subproject commit 772bd430d58b907239a92c09ed8d94ef7790a827