Add heroku/heroku:24 and heroku/heroku:24-build as multiarch images (h…

…eroku#245)

* Add heroku-24 and heroku-24-build as multiarch

* Don't create /app or /workspace or set them as $HOME

* Regenerate installed-packages.txt

* Drop mtools. It was a dependency of syslinux, which is no longer installed

* Also write the package list for build variants

* Regenerate installed packages for heroku-24

* Drop ghostscript, libgs10, libgs10-dev, gsfonts

* Add 24 to the CI matrix

* Enable the multiarch driver

* Don't try to push cnb variants for heroku-24

* Drop language-pack-en

* Use --load for buildx builds

* Generate locale for en_us

* Rework scripting to handle multiarch, single arch, linux docker, and docker desktop

* If we're not pushing, we're loading. buildx requires us to be explicit

* Be explicit about what platform to build for 24 single architecture builds

* Detect docker-container driver and containerd snapshotter rather than Docker Desktop

* Regenerate installed packages

* Print detailed usage context

* Fix erroneous argument check

* Drop brz and python3-dev

* Update installed packages list

* Drop socat and telnet

* Drop explicit ed dependency

* Regenerate package lists for heroku-24

* Generate package lists for all architectures when possible

* Add CNB Spec reference

Co-authored-by: Ed Morley <[email protected]>

* Add CNB Spec reference

Co-authored-by: Ed Morley <[email protected]>

* Address formatting, output, and exit code feedback

* Use bash double brackets / parens

* Use RUN --mount to drop COPY step

* Drop redundant label

* Separate build and run users for heroku-24

* Add ed, socat, telnet back into the package list for heroku-24

* Drop unused internalTag

* Update to libmagickcore-6.q16-7-extra and regenerate package lists

* Fix multi-arch conditional

* Use double brackets for publishing script

* Drop unnecessary $ in arithmetic

* Use noble channel for postgres, now that it is available

* Add a comment explaining locale-gen

* Update imagemagick policy to match ubuntu defaults

* Reorder magick policy to match more closely to ubuntu

* Publish to public temp tags instead of private

* Drop shared cache weak secret in imagemagick config

* Unpublish temp tags after CI

* Fixup unpublish step

* Stop generating imagemagick policy, since ubuntu's is fine

* Fix syntax for docker hub delete curl command

* Update tag deletion to v2 url

* Fix error handling during unpublish

* Update heroku-24 package lists

* Regenerate heroku-24-build package lists

* Update incorrect usage messaging

Signed-off-by: Josh W Lewis <[email protected]>

* Fix some leading spaces to tabs in bin/build.sh

* Update curl and jq usage to match languages standards

Co-authored-by: Ed Morley <[email protected]>

* Update curl and jq usage to match languages standards

Co-authored-by: Ed Morley <[email protected]>

* Construct targetTagName only once, and reindent bin/unpublish-tags.sh

---------

Signed-off-by: Josh W Lewis <[email protected]>
Co-authored-by: Ed Morley <[email protected]>
Co-authored-by: David Zülke <[email protected]>

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        stack-version: ["20", "22"]
+        stack-version: ["20", "22", "24"]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -50,8 +50,13 @@ jobs:
             exit 1
           fi
       - name: Publish to image registries
-        run: bin/publish-to-registries.sh
+        run: |-
+          docker buildx create --use
+          bin/publish-to-registries.sh
         if: success() && (github.ref_name == 'main' || github.ref_type == 'tag')
+      - name: Unpublish temp tags from this run
+        run: bin/unpublish-tags.sh
+        if: always()
       - name: Convert docker image and release to Heroku staging
         run: bin/convert-and-publish-to-heroku.sh
         if: success() && github.ref_type == 'tag'

bin/build.sh

@@ -5,46 +5,159 @@ set -euo pipefail
 cd "$(dirname "${BASH_SOURCE[0]}")/.."
 . bin/stack-helpers.sh
 
-[ $# -eq 1 ] || abort "usage: $(basename "${BASH_SOURCE[0]}") STACK_VERSION"
+REPO="heroku/heroku"
+STACK_VERSION=${1:-"NAN"}
+PUBLISH_SUFFIX=${2:-}
+BASE_NAME=$(basename "${BASH_SOURCE[0]}")
 
-STACK_VERSION=$1
+print_usage(){
+	>&2 echo "usage: ${BASE_NAME}  STACK_VERSION [PUBLISH_SUFFIX]"
+	>&2 cat <<-EOF
+
+		This script builds heroku base images and writes package lists. It builds
+		multi-arch images for heroku-24 and newer, and amd64 images for heroku-22 and
+		older. It works in the following scenarios:
+
+		Local builds with Docker Desktop and the 'containerd' snapshotter. In this
+		mode, all resulting images will be loaded into the local container store.
+		The 'default' and 'docker-container' drivers both work in this mode.
+		Note that the 'containerd' snapshotter is not compatible with 'pack'.
+
+		For CI tests and package list generation with Docker and the 'default'
+		Docker driver. In this mode, resulting images will be loaded into the
+		local container store, the package lists generated, but only for amd64.
+		The 'default' Docker driver is not able to store/retreive multi-arch images
+		locally with the default snapshotter, and the 'containerd' snapshotter is
+		only available with Docker Desktop. The 'docker-container' driver will not
+		work in this mode (it can't load any images from the default local store).
+
+		Publishing images in CI with Docker and the 'docker-container'
+		driver. Pass in a REPO and PUBLISH_SUFFIX argument to publish images
+		directly during the build. Since Docker is unable to store/reference
+		multi-arch images locally, the publish process involves building+pushing
+		an image to a disposable tag, then retagging it. The 'default' Docker
+		driver will not work in this mode (it can't build cross-architecture).
+	EOF
+}
+
+[[ $STACK_VERSION =~ ^[0-9]+$ ]] || (>&2 print_usage && abort "fatal: invalid STACK_VERSION")
+
+have_docker_container_driver=
+if (docker buildx inspect; true) | grep -q 'Driver:\s*docker-container$'; then
+	have_docker_container_driver=1
+fi
+
+have_containerd_snapshotter=
+if docker info -f "{{ .DriverStatus }}" | grep -qF "io.containerd.snapshotter."; then have_containerd_snapshotter=1; fi
+
+
+if (( STACK_VERSION <= 22 )); then
+	# heroku/heroku:22 and prior images do not support multiple chip
+	# architectures or multi-arch images. Instead, they are amd64 only.
+	DOCKER_ARGS=("build" "--platform=linux/amd64")
+	# heroku/heroku:22 and prior images need separate *cnb* variants that
+	# add compatibility for Cloud Native Buildpacks.
+	VARIANTS=("-build:" "-cnb:" "-cnb-build:-build")
+else
+	# heroku/heroku:24 images and beyond are multi-arch (amd64+arm64) images.
+	# Due to weak feature support parity between Docker on Linux and Docker
+	# Desktop building and publishing across platforms has caveats (see the
+	# top of this file).
+	if [[ $have_containerd_snapshotter ]] || { [[ $PUBLISH_SUFFIX ]] && [[ $have_docker_container_driver ]]; }; then
+		DOCKER_ARGS=("buildx" "build" "--platform=linux/amd64,linux/arm64")
+	elif [[ ! $PUBLISH_SUFFIX ]] && [[ ! $have_docker_container_driver ]]; then
+		DOCKER_ARGS=("buildx" "build" "--platform=linux/amd64")
+		>&2 echo "WARNING: heroku-24 and newer images are multi-arch images," \
+			"but this script is building single architecture images" \
+			"due to limitations of the current platform." \
+			"To build a multi-arch image, enable the 'containerd'" \
+			"snapshotter in Docker Desktop and/or use a 'docker-container'" \
+			"Docker BuildKit driver."
+	else
+		>&2 echo "ERROR: Can't build images with this configuration. Enable" \
+			"the 'containerd' snapshotter in Docker Desktop, enable" \
+			"the 'docker-container' driver in Docker, or use this script" \
+			"in build-only mode (don't provide PUBLISH_SUFFIX argument)."
+		abort 1
+	fi
+	# heroku/heroku:24 and beyond images include CNB specific
+	# modifications, so separate *cnb* variants are not created.
+	VARIANTS=("-build:")
+fi
+
+if [[ $PUBLISH_SUFFIX ]]; then
+	# If there is a tag suffix, this script is pushing to a remote registry.
+	DOCKER_ARGS+=("--push")
+else
+	# Otherwise, load the image into the local image store.
+	DOCKER_ARGS+=("--load")
+fi
 
 write_package_list() {
-    local image_tag="$1"
-    local output_file="${2}/installed-packages.txt"
-    echo '# List of packages present in the final image. Regenerate using bin/build.sh' > "$output_file"
-    docker run --rm "$image_tag" dpkg-query --show --showformat='${Package}\n' >> "$output_file"
+	local image_tag="$1"
+	local dockerfile_dir="$2"
+
+	# Extract the stack version from the dockerfile_dir variable (e.g., heroku-24)
+	local stack_version
+	stack_version=$(echo "$dockerfile_dir" | sed -n 's/^heroku-\([0-9]*\).*$/\1/p')
+
+	local archs=("amd64")
+	# heroku-24 and newer are multiarch. If containerd is available,
+	# the package list for each architecture can be generated.
+	if (( stack_version >= 24 )); then
+		if [[ $have_containerd_snapshotter ]]; then
+			archs+=(arm64)
+		else
+			>&2 echo "WARNING: Generating package list for single architecture." \
+				"Use the 'containerd' snapshotter to generate package lists" \
+				"for all architectures."
+		fi
+	fi
+	local output_file=""
+	for arch in "${archs[@]}"; do
+		if (( stack_version >= 24 )); then
+			output_file="${dockerfile_dir}/installed-packages-${arch}.txt"
+		else
+			output_file="${dockerfile_dir}/installed-packages.txt"
+		fi
+		display "Generating package list: ${output_file}"
+		echo "# List of packages present in the final image. Regenerate using bin/build.sh" > "$output_file"
+		docker run --rm --platform="linux/${arch}" "$image_tag" dpkg-query --show --showformat='${Package}\n' >> "$output_file"
+	done
 }
 
-RUN_IMAGE_TAG="heroku/heroku:${STACK_VERSION}"
+RUN_IMAGE_TAG="${REPO}:${STACK_VERSION}${PUBLISH_SUFFIX}"
 RUN_DOCKERFILE_DIR="heroku-${STACK_VERSION}"
-[[ -d "${RUN_DOCKERFILE_DIR}" ]] || abort "fatal: directory ${RUN_DOCKERFILE_DIR} not found"
-display "Building ${RUN_DOCKERFILE_DIR} / ${RUN_IMAGE_TAG} Heroku run image"
-docker build --pull --tag "${RUN_IMAGE_TAG}" "${RUN_DOCKERFILE_DIR}" | indent
-write_package_list "${RUN_IMAGE_TAG}" "${RUN_DOCKERFILE_DIR}"
 
-# The --pull option is not used for variants to ensure they are based on the
-# run image built above, rather than the one last published to Docker Hub.
-
-BUILD_IMAGE_TAG="${RUN_IMAGE_TAG}-build"
-BUILD_DOCKERFILE_DIR="${RUN_DOCKERFILE_DIR}-build"
-display "Building ${BUILD_DOCKERFILE_DIR} / ${BUILD_IMAGE_TAG} Heroku build image"
-docker build --tag "$BUILD_IMAGE_TAG" "$BUILD_DOCKERFILE_DIR" | indent
-write_package_list "$BUILD_IMAGE_TAG" "$BUILD_DOCKERFILE_DIR"
+[[ -d "${RUN_DOCKERFILE_DIR}" ]] || abort "fatal: directory ${RUN_DOCKERFILE_DIR} not found"
+display "Building ${RUN_DOCKERFILE_DIR} / ${RUN_IMAGE_TAG} image"
+# The --pull option is used for the run image, so that the latest updates
+# from upstream ubuntu images are included.
+docker "${DOCKER_ARGS[@]}" --pull \
+	--tag "${RUN_IMAGE_TAG}" "${RUN_DOCKERFILE_DIR}" | indent
+	write_package_list "${RUN_IMAGE_TAG}" "${RUN_DOCKERFILE_DIR}"
 
-# write_package_list is not needed for *cnb* variants, as they don't install
-# any additional packages over their non-*cnb* counterparts.
+	for VARIANT in "${VARIANTS[@]}"; do
+		VARIANT_NAME=$(echo "$VARIANT" | cut -d ":" -f 1)
+		DEPENDENCY_NAME=$(echo "$VARIANT" | cut -d ":" -f 2)
+		VARIANT_IMAGE_TAG="${REPO}:${STACK_VERSION}${VARIANT_NAME}${PUBLISH_SUFFIX}"
+		VARIANT_DOCKERFILE_DIR="heroku-${STACK_VERSION}${VARIANT_NAME}"
+		DEPENDENCY_IMAGE_TAG="${REPO}:${STACK_VERSION}${DEPENDENCY_NAME}${PUBLISH_SUFFIX}"
 
-CNB_RUN_IMAGE_TAG="${RUN_IMAGE_TAG}-cnb"
-CNB_RUN_DOCKERFILE_DIR="${RUN_DOCKERFILE_DIR}-cnb"
-display "Building ${CNB_RUN_DOCKERFILE_DIR} / ${CNB_RUN_IMAGE_TAG} CNB run image"
-docker build --tag "$CNB_RUN_IMAGE_TAG" "$CNB_RUN_DOCKERFILE_DIR" | indent
+		[[ -d "${VARIANT_DOCKERFILE_DIR}" ]] || abort "fatal: directory ${VARIANT_DOCKERFILE_DIR} not found"
+		display "Building ${VARIANT_DOCKERFILE_DIR} / ${VARIANT_IMAGE_TAG} image"
+		# The --pull option is not used for variants since they depend on images
+		# built earlier in this script.
+		docker "${DOCKER_ARGS[@]}" --build-arg "BASE_IMAGE=${DEPENDENCY_IMAGE_TAG}" \
+			--tag "${VARIANT_IMAGE_TAG}" "${VARIANT_DOCKERFILE_DIR}" | indent
 
-CNB_BUILD_IMAGE_TAG="${RUN_IMAGE_TAG}-cnb-build"
-CNB_BUILD_DOCKERFILE_DIR="${RUN_DOCKERFILE_DIR}-cnb-build"
-display "Building ${CNB_BUILD_DOCKERFILE_DIR} / ${CNB_BUILD_IMAGE_TAG} CNB build image"
-docker build --tag "$CNB_BUILD_IMAGE_TAG" "$CNB_BUILD_DOCKERFILE_DIR" | indent
+	# generate the package list for non-cnb variants. cnb variants don't
+	# influence the list of installed packages.
+	if [[ ! "$VARIANT_NAME" = -cnb* ]]; then
+		write_package_list "$VARIANT_IMAGE_TAG" "$VARIANT_DOCKERFILE_DIR"
+	fi
+done
 
 display "Size breakdown..."
 docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" \
-    | grep -E "(ubuntu|heroku)" | sed '1!G;h;$!d' | indent
+	| grep -E "(ubuntu|heroku)" | sed '1!G;h;$!d' | indent

bin/publish-to-registries.sh

@@ -3,8 +3,6 @@
 set -euo pipefail
 set -x
 
-bin/build.sh "${STACK_VERSION}"
-
 (
   # Disable tracing (until the end of this subshell) to prevent logging registry tokens.
   set +x
@@ -14,25 +12,34 @@ bin/build.sh "${STACK_VERSION}"
 )
 
 push_group() {
-    local targetTagBase="$1"
-    local targetTagSuffix="$2"
-    for variant in "" "-build" "-cnb" "-cnb-build"; do
-      source="${publicTag}${variant}"
-      target="${targetTagBase}${variant}${targetTagSuffix}"
+    local tagBase="$1"
+    local sourceTagSuffix="$2"
+    local targetTagSuffix="$3"
+    variants=("" "-build")
+    if (( STACK_VERSION <= 22 )); then
+        variants+=("-cnb" "-cnb-build")
+    fi
+    for variant in "${variants[@]}"; do
+      source="${tagBase}${variant}${sourceTagSuffix}"
+      target="${tagBase}${variant}${targetTagSuffix}"
       docker tag "${source}" "${target}"
       docker push "${target}"
     done
 }
 
+tempTagSuffix=".temp-${GITHUB_RUN_ID}"
+# build+push to a temporary tag (e.g. heroku/heroku:22.temp_12345678)
+bin/build.sh "${STACK_VERSION}" "${tempTagSuffix}"
+
 publicTag="heroku/heroku:${STACK_VERSION}"
 
 # Push nightly tags to Docker Hub (e.g. heroku/heroku:22.nightly)
-push_group "${publicTag}" ".nightly"
+push_group "${publicTag}" "${tempTagSuffix}" ".nightly"
 
-if [ "$GITHUB_REF_TYPE" == 'tag' ]; then
+if [[ "$GITHUB_REF_TYPE" == 'tag' ]]; then
   # Push release tags to Docker Hub (e.g. heroku/heroku:22.v99)
-  push_group "${publicTag}" ".${GITHUB_REF_NAME}"
+  push_group "${publicTag}" "${tempTagSuffix}" ".${GITHUB_REF_NAME}"
 
   # Push latest/no-suffix tags to Docker Hub (e.g. heroku/heroku:22)
-  push_group "${publicTag}" ""
+  push_group "${publicTag}" "${tempTagSuffix}" ""
 fi

bin/unpublish-tags.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+dockerhub_token=$(curl -sS -f --retry 3 --retry-connrefused --connect-timeout 5 --max-time 30 -H "Content-Type: application/json" -X POST -d "{\"username\": \"${DOCKER_HUB_USERNAME}\", \"password\": \"${DOCKER_HUB_TOKEN}\"}" https://hub.docker.com/v2/users/login/ | jq --exit-status -r .token)
+
+unpublish_group() {
+	local stackVersion="$1"
+	local targetTagSuffix="$2"
+	local status=0
+	variants=("" "-build")
+	if (( stackVersion <= 22 )); then
+		variants+=("-cnb" "-cnb-build")
+	fi
+	for variant in "${variants[@]}"; do
+		targetTagName="${stackVersion}${variant}${targetTagSuffix}"
+		echo "Deleting heroku/heroku:${targetTagName}"
+		response=$(curl -sS --retry 3 --retry-connrefused --connect-timeout 5 --max-time 30 -X DELETE \
+			-H "Authorization: JWT ${dockerhub_token}" \
+			"https://hub.docker.com/v2/namespaces/heroku/repositories/heroku/tags/${targetTagName}"
+		)
+
+		if [[ -z $response ]]; then
+			>&2 echo "Deleted."
+		elif [[ $response =~ "tag not found" ]]; then
+			>&2 echo "Tag does not exist."
+		else
+			>&2 echo "Couldn't delete. Response: ${response}"
+			status=22
+		fi
+	done
+	return $status
+}
+
+stackVersion="${1:-$STACK_VERSION}"
+tempTagSuffix="${2:-".temp-$GITHUB_RUN_ID"}"
+# delete each tag in a group on Docker Hub.
+unpublish_group "${stackVersion}" "${tempTagSuffix}"

heroku-20-build/Dockerfile

@@ -1,3 +1,4 @@
-FROM heroku/heroku:20
+ARG BASE_IMAGE=heroku/heroku:20
+FROM $BASE_IMAGE
 COPY setup.sh /tmp/setup.sh
 RUN /tmp/setup.sh

heroku-20-cnb-build/Dockerfile

@@ -1,4 +1,5 @@
-FROM heroku/heroku:20-build
+ARG BASE_IMAGE=heroku/heroku:20-build
+FROM $BASE_IMAGE
 
 RUN groupadd heroku --gid 1000 && \
   useradd heroku -u 1000 -g 1000 -s /bin/bash -m

heroku-20-cnb/Dockerfile

@@ -1,4 +1,5 @@
-FROM heroku/heroku:20
+ARG BASE_IMAGE=heroku/heroku:20
+FROM $BASE_IMAGE
 
 RUN ln -s /workspace /app
 

heroku-22-build/Dockerfile

@@ -1,3 +1,4 @@
-FROM heroku/heroku:22
+ARG BASE_IMAGE=heroku/heroku:22
+FROM $BASE_IMAGE
 COPY setup.sh /tmp/setup.sh
 RUN /tmp/setup.sh

heroku-22-cnb-build/Dockerfile

@@ -1,4 +1,5 @@
-FROM heroku/heroku:22-build
+ARG BASE_IMAGE=heroku/heroku:22-build
+FROM $BASE_IMAGE
 
 RUN groupadd heroku --gid 1000 && \
   useradd heroku -u 1000 -g 1000 -s /bin/bash -m

heroku-22-cnb/Dockerfile

@@ -1,4 +1,5 @@
-FROM heroku/heroku:22
+ARG BASE_IMAGE=heroku/heroku:22
+FROM $BASE_IMAGE
 
 RUN ln -s /workspace /app
 

heroku-24-build/.dockerignore

@@ -0,0 +1 @@
+installed-packages*.txt

heroku-24-build/Dockerfile

@@ -0,0 +1,10 @@
+ARG BASE_IMAGE=heroku/heroku:24
+FROM $BASE_IMAGE
+USER root
+RUN --mount=target=/build /build/setup.sh
+
+# https://github.com/buildpacks/spec/blob/main/platform.md#build-image
+USER 1002
+ENV CNB_USER_ID=1002
+ENV CNB_GROUP_ID=1000
+ENV CNB_STACK_ID "heroku-24"