Merge branch 'multiarch-push' into multiarch-push-robuust

# Conflicts:
#	.github/workflows/ci.yml
#	bin/build.sh
#	heroku-22-build/Dockerfile

bin/build.sh

@@ -5,49 +5,153 @@ set -euo pipefail
 cd "$(dirname "${BASH_SOURCE[0]}")/.."
 . bin/stack-helpers.sh
 
-[ $# -eq 1 ] || abort "usage: $(basename "${BASH_SOURCE[0]}") STACK_VERSION"
+REPO="ghcr.io/robuust/heroku"
+STACK_VERSION=${1:-"NAN"}
+PUBLISH_SUFFIX=${2:-}
+BASE_NAME=$(basename "${BASH_SOURCE[0]}")
 
-STACK_VERSION=$1
+print_usage(){
+	>&2 echo "usage: ${BASE_NAME}  STACK_VERSION [PUBLISH_SUFFIX]"
+	>&2 cat <<-EOF
 
-write_package_list() {
-    local image_tag="$1"
-    local output_file="${2}/installed-packages.txt"
-    echo '# List of packages present in the final image. Regenerate using bin/build.sh' > "$output_file"
-    docker run --rm "$image_tag" dpkg-query --show --showformat='${Package}\n' >> "$output_file"
+		This script builds heroku base images and writes package lists. It builds
+		multi-arch images for heroku-24 and newer, and amd64 images for heroku-22 and
+		older. It works in the following scenarios:
+
+		Local builds with Docker Desktop and the 'containerd' snapshotter. In this
+		mode, all resulting images will be loaded into the local container store.
+		The 'default' and 'docker-container' drivers both work in this mode.
+		Note that the 'containerd' snapshotter is not compatible with 'pack'.
+
+		For CI tests and package list generation with Docker and the 'default'
+		Docker driver. In this mode, resulting images will be loaded into the
+		local container store, the package lists generated, but only for amd64.
+		The 'default' Docker driver is not able to store/retreive multi-arch images
+		locally with the default snapshotter, and the 'containerd' snapshotter is
+		only available with Docker Desktop. The 'docker-container' driver will not
+		work in this mode (it can't load any images from the default local store).
+
+		Publishing images in CI with Docker and the 'docker-container'
+		driver. Pass in a REPO and PUBLISH_SUFFIX argument to publish images
+		directly during the build. Since Docker is unable to store/reference
+		multi-arch images locally, the publish process involves building+pushing
+		an image to a disposable tag, then retagging it. The 'default' Docker
+		driver will not work in this mode (it can't build cross-architecture).
+	EOF
 }
 
 # Create buildx driver
 docker buildx create --use
 
-RUN_IMAGE_TAG="ghcr.io/robuust/heroku:${STACK_VERSION}"
-RUN_DOCKERFILE_DIR="heroku-${STACK_VERSION}"
-[[ -d "${RUN_DOCKERFILE_DIR}" ]] || abort "fatal: directory ${RUN_DOCKERFILE_DIR} not found"
-display "Building ${RUN_DOCKERFILE_DIR} / ${RUN_IMAGE_TAG} Heroku run image"
-docker buildx build --pull --platform=linux/arm64,linux/amd64 --tag "${RUN_IMAGE_TAG}" --push "${RUN_DOCKERFILE_DIR}" | indent
-write_package_list "${RUN_IMAGE_TAG}" "${RUN_DOCKERFILE_DIR}"
+[[ $STACK_VERSION =~ ^[0-9]+$ ]] || (>&2 print_usage && abort "fatal: invalid STACK_VERSION")
+
+have_docker_container_driver=
+if (docker buildx inspect; true) | grep -q 'Driver:\s*docker-container$'; then
+	have_docker_container_driver=1
+fi
+
+have_containerd_snapshotter=
+if docker info -f "{{ .DriverStatus }}" | grep -qF "io.containerd.snapshotter."; then have_containerd_snapshotter=1; fi
+
+
+if (( STACK_VERSION <= 22 )); then
+	# heroku/heroku:22 and prior images need separate *cnb* variants that
+	# add compatibility for Cloud Native Buildpacks.
+	VARIANTS=("-build:" "-cnb:" "-cnb-build:-build")
+else
+	# heroku/heroku:24 and beyond images include CNB specific
+	# modifications, so separate *cnb* variants are not created.
+	VARIANTS=("-build:")
+fi
 
-# The --pull option is not used for variants to ensure they are based on the
-# run image built above, rather than the one last published to Docker Hub.
+# Due to weak feature support parity between Docker on Linux and Docker
+# Desktop building and publishing across platforms has caveats (see the
+# top of this file).
+if [[ $have_containerd_snapshotter ]] || { [[ $PUBLISH_SUFFIX ]] && [[ $have_docker_container_driver ]]; }; then
+	DOCKER_ARGS=("buildx" "build" "--platform=linux/amd64,linux/arm64")
+elif [[ ! $PUBLISH_SUFFIX ]] && [[ ! $have_docker_container_driver ]]; then
+	DOCKER_ARGS=("buildx" "build" "--platform=linux/amd64")
+	>&2 echo "WARNING: heroku-24 and newer images are multi-arch images," \
+		"but this script is building single architecture images" \
+		"due to limitations of the current platform." \
+		"To build a multi-arch image, enable the 'containerd'" \
+		"snapshotter in Docker Desktop and/or use a 'docker-container'" \
+		"Docker BuildKit driver."
+else
+	>&2 echo "ERROR: Can't build images with this configuration. Enable" \
+		"the 'containerd' snapshotter in Docker Desktop, enable" \
+		"the 'docker-container' driver in Docker, or use this script" \
+		"in build-only mode (don't provide PUBLISH_SUFFIX argument)."
+	abort 1
+fi
 
-BUILD_IMAGE_TAG="${RUN_IMAGE_TAG}-build"
-BUILD_DOCKERFILE_DIR="${RUN_DOCKERFILE_DIR}-build"
-display "Building ${BUILD_DOCKERFILE_DIR} / ${BUILD_IMAGE_TAG} Heroku build image"
-docker buildx build --platform=linux/arm64,linux/amd64 --tag "$BUILD_IMAGE_TAG" --push "$BUILD_DOCKERFILE_DIR" | indent
-write_package_list "$BUILD_IMAGE_TAG" "$BUILD_DOCKERFILE_DIR"
+if [[ $PUBLISH_SUFFIX ]]; then
+	# If there is a tag suffix, this script is pushing to a remote registry.
+	DOCKER_ARGS+=("--push")
+else
+	# Otherwise, load the image into the local image store.
+	DOCKER_ARGS+=("--load")
+fi
+
+write_package_list() {
+	local image_tag="$1"
+	local dockerfile_dir="$2"
+
+	# Extract the stack version from the dockerfile_dir variable (e.g., heroku-24)
+	local stack_version
+	stack_version=$(echo "$dockerfile_dir" | sed -n 's/^heroku-\([0-9]*\).*$/\1/p')
+
+	local archs=("amd64")
+	# If containerd is available,
+	# the package list for each architecture can be generated.
+	if [[ $have_containerd_snapshotter ]]; then
+		archs+=(arm64)
+	else
+		>&2 echo "WARNING: Generating package list for single architecture." \
+			"Use the 'containerd' snapshotter to generate package lists" \
+			"for all architectures."
+	fi
+	local output_file=""
+	for arch in "${archs[@]}"; do
+		output_file="${dockerfile_dir}/installed-packages-${arch}.txt"
+		display "Generating package list: ${output_file}"
+		echo "# List of packages present in the final image. Regenerate using bin/build.sh" > "$output_file"
+		docker run --rm --platform="linux/${arch}" "$image_tag" dpkg-query --show --showformat='${Package}\n' >> "$output_file"
+	done
+}
+
+RUN_IMAGE_TAG="${REPO}:${STACK_VERSION}${PUBLISH_SUFFIX}"
+RUN_DOCKERFILE_DIR="heroku-${STACK_VERSION}"
+
+[[ -d "${RUN_DOCKERFILE_DIR}" ]] || abort "fatal: directory ${RUN_DOCKERFILE_DIR} not found"
+display "Building ${RUN_DOCKERFILE_DIR} / ${RUN_IMAGE_TAG} image"
+# The --pull option is used for the run image, so that the latest updates
+# from upstream ubuntu images are included.
+docker "${DOCKER_ARGS[@]}" --pull \
+	--tag "${RUN_IMAGE_TAG}" "${RUN_DOCKERFILE_DIR}" | indent
+	write_package_list "${RUN_IMAGE_TAG}" "${RUN_DOCKERFILE_DIR}"
 
-# write_package_list is not needed for *cnb* variants, as they don't install
-# any additional packages over their non-*cnb* counterparts.
+	for VARIANT in "${VARIANTS[@]}"; do
+		VARIANT_NAME=$(echo "$VARIANT" | cut -d ":" -f 1)
+		DEPENDENCY_NAME=$(echo "$VARIANT" | cut -d ":" -f 2)
+		VARIANT_IMAGE_TAG="${REPO}:${STACK_VERSION}${VARIANT_NAME}${PUBLISH_SUFFIX}"
+		VARIANT_DOCKERFILE_DIR="heroku-${STACK_VERSION}${VARIANT_NAME}"
+		DEPENDENCY_IMAGE_TAG="${REPO}:${STACK_VERSION}${DEPENDENCY_NAME}${PUBLISH_SUFFIX}"
 
-# CNB_RUN_IMAGE_TAG="${RUN_IMAGE_TAG}-cnb"
-# CNB_RUN_DOCKERFILE_DIR="${RUN_DOCKERFILE_DIR}-cnb"
-# display "Building ${CNB_RUN_DOCKERFILE_DIR} / ${CNB_RUN_IMAGE_TAG} CNB run image"
-# docker buildx build --platform=linux/arm64,linux/amd64 --tag "$CNB_RUN_IMAGE_TAG" --push "$CNB_RUN_DOCKERFILE_DIR" | indent
+		[[ -d "${VARIANT_DOCKERFILE_DIR}" ]] || abort "fatal: directory ${VARIANT_DOCKERFILE_DIR} not found"
+		display "Building ${VARIANT_DOCKERFILE_DIR} / ${VARIANT_IMAGE_TAG} image"
+		# The --pull option is not used for variants since they depend on images
+		# built earlier in this script.
+		docker "${DOCKER_ARGS[@]}" --build-arg "BASE_IMAGE=${DEPENDENCY_IMAGE_TAG}" \
+			--tag "${VARIANT_IMAGE_TAG}" "${VARIANT_DOCKERFILE_DIR}" | indent
 
-# CNB_BUILD_IMAGE_TAG="${RUN_IMAGE_TAG}-cnb-build"
-# CNB_BUILD_DOCKERFILE_DIR="${RUN_DOCKERFILE_DIR}-cnb-build"
-# display "Building ${CNB_BUILD_DOCKERFILE_DIR} / ${CNB_BUILD_IMAGE_TAG} CNB build image"
-# docker buildx build --platform=linux/arm64,linux/amd64 --tag "$CNB_BUILD_IMAGE_TAG" --push "$CNB_BUILD_DOCKERFILE_DIR" | indent
+	# generate the package list for non-cnb variants. cnb variants don't
+	# influence the list of installed packages.
+	if [[ ! "$VARIANT_NAME" = -cnb* ]]; then
+		write_package_list "$VARIANT_IMAGE_TAG" "$VARIANT_DOCKERFILE_DIR"
+	fi
+done
 
 display "Size breakdown..."
 docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" \
-    | grep -E "(ubuntu|heroku)" | sed '1!G;h;$!d' | indent
+	| grep -E "(ubuntu|heroku)" | sed '1!G;h;$!d' | indent

bin/publish-to-registries.sh

@@ -9,21 +9,21 @@ set -x
 
   echo "Logging into Docker Hub..."
   echo "${DOCKER_HUB_TOKEN}" | docker login -u "${DOCKER_HUB_USERNAME}" --password-stdin
-
-  echo "Logging into internal container registry..."
-  curl -sSf --retry 3 -X POST "$ID_SERVICE_TOKEN_ENDPOINT" -d "{\"username\":\"${ID_SERVICE_USERNAME}\",\"password\":\"${ID_SERVICE_PASSWORD}\"}" \
-    | jq -er ".raw_id_token" \
-    | docker login "$INTERNAL_REGISTRY_HOST" -u "$INTERNAL_REGISTRY_USERNAME" --password-stdin
 )
 
 bin/build.sh "${STACK_VERSION}"
 
 push_group() {
-    local targetTagBase="$1"
-    local targetTagSuffix="$2"
-    for variant in "" "-build" "-cnb" "-cnb-build"; do
-      source="${publicTag}${variant}"
-      target="${targetTagBase}${variant}${targetTagSuffix}"
+    local tagBase="$1"
+    local sourceTagSuffix="$2"
+    local targetTagSuffix="$3"
+    variants=("" "-build")
+    if (( STACK_VERSION <= 22 )); then
+        variants+=("-cnb" "-cnb-build")
+    fi
+    for variant in "${variants[@]}"; do
+      source="${tagBase}${variant}${sourceTagSuffix}"
+      target="${tagBase}${variant}${targetTagSuffix}"
       chmod +r "$HOME"/.docker/config.json
       docker container run --rm --net host \
         -v regctl-conf:/home/appuser/.regctl/ \
@@ -32,27 +32,19 @@ push_group() {
     done
 }
 
-date=$(date -u '+%Y-%m-%d-%H.%M.%S')
-publicTag="heroku/heroku:${STACK_VERSION}"
-privateTag="heroku/heroku-private:${STACK_VERSION}"
-internalTag="${INTERNAL_REGISTRY_HOST}/s/${ID_SERVICE_USERNAME}/heroku:${STACK_VERSION}"
-
-# Push nightly tags to dockerhub (e.g. heroku/heroku:22.nightly)
-push_group "${publicTag}" ".nightly"
+tempTagSuffix=".temp-${GITHUB_RUN_ID}"
+# build+push to a temporary tag (e.g. heroku/heroku:22.temp_12345678)
+bin/build.sh "${STACK_VERSION}" "${tempTagSuffix}"
 
-# Push date tags to private dockerhub (e.g. heroku/heroku-private:22.2022-06-01-17.00.00)
-push_group "${privateTag}" ".${date}"
-
-if [ "$GITHUB_REF_TYPE" == 'tag' ]; then
-  # Push release tags to dockerhub (e.g. heroku/heroku:22.v99)
-  push_group "${publicTag}" ".${GITHUB_REF_NAME}"
+publicTag="heroku/heroku:${STACK_VERSION}"
 
-  # Push release tags to internal registry
-  push_group "${internalTag}" ".${GITHUB_REF_NAME}"
+# Push nightly tags to Docker Hub (e.g. heroku/heroku:22.nightly)
+push_group "${publicTag}" "${tempTagSuffix}" ".nightly"
 
-  # Push latest/no-suffix tags to dockerhub (e.g. heroku/heroku:22)
-  push_group "${publicTag}" ""
+if [[ "$GITHUB_REF_TYPE" == 'tag' ]]; then
+  # Push release tags to Docker Hub (e.g. heroku/heroku:22.v99)
+  push_group "${publicTag}" "${tempTagSuffix}" ".${GITHUB_REF_NAME}"
 
-  # Push latest/no-suffix tags to internal registry
-  push_group "${internalTag}" ""
+  # Push latest/no-suffix tags to Docker Hub (e.g. heroku/heroku:22)
+  push_group "${publicTag}" "${tempTagSuffix}" ""
 fi

bin/unpublish-tags.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+dockerhub_token=$(curl -sS -f --retry 3 --retry-connrefused --connect-timeout 5 --max-time 30 -H "Content-Type: application/json" -X POST -d "{\"username\": \"${DOCKER_HUB_USERNAME}\", \"password\": \"${DOCKER_HUB_TOKEN}\"}" https://hub.docker.com/v2/users/login/ | jq --exit-status -r .token)
+
+unpublish_group() {
+	local stackVersion="$1"
+	local targetTagSuffix="$2"
+	local status=0
+	variants=("" "-build")
+	if (( stackVersion <= 22 )); then
+		variants+=("-cnb" "-cnb-build")
+	fi
+	for variant in "${variants[@]}"; do
+		targetTagName="${stackVersion}${variant}${targetTagSuffix}"
+		echo "Deleting heroku/heroku:${targetTagName}"
+		response=$(curl -sS --retry 3 --retry-connrefused --connect-timeout 5 --max-time 30 -X DELETE \
+			-H "Authorization: JWT ${dockerhub_token}" \
+			"https://hub.docker.com/v2/namespaces/heroku/repositories/heroku/tags/${targetTagName}"
+		)
+
+		if [[ -z $response ]]; then
+			>&2 echo "Deleted."
+		elif [[ $response =~ "tag not found" ]]; then
+			>&2 echo "Tag does not exist."
+		else
+			>&2 echo "Couldn't delete. Response: ${response}"
+			status=22
+		fi
+	done
+	return $status
+}
+
+stackVersion="${1:-$STACK_VERSION}"
+tempTagSuffix="${2:-".temp-$GITHUB_RUN_ID"}"
+# delete each tag in a group on Docker Hub.
+unpublish_group "${stackVersion}" "${tempTagSuffix}"

heroku-20-build/Dockerfile

@@ -1,3 +1,4 @@
-FROM heroku/heroku:20
+ARG BASE_IMAGE=heroku/heroku:20
+FROM $BASE_IMAGE
 COPY setup.sh /tmp/setup.sh
 RUN /tmp/setup.sh

heroku-20-cnb-build/Dockerfile

@@ -1,4 +1,5 @@
-FROM heroku/heroku:20-build
+ARG BASE_IMAGE=heroku/heroku:20-build
+FROM $BASE_IMAGE
 
 RUN groupadd heroku --gid 1000 && \
   useradd heroku -u 1000 -g 1000 -s /bin/bash -m
@@ -8,8 +9,10 @@ RUN mkdir /app && \
 
 ENV CNB_USER_ID=1000
 ENV CNB_GROUP_ID=1000
-
 ENV CNB_STACK_ID "heroku-20"
+ENV CNB_TARGET_OS="linux"
+ENV CNB_TARGET_ARCH="amd64"
+
 LABEL io.buildpacks.stack.id="heroku-20"
 
 USER heroku

heroku-20-cnb/Dockerfile

@@ -1,4 +1,5 @@
-FROM heroku/heroku:20
+ARG BASE_IMAGE=heroku/heroku:20
+FROM $BASE_IMAGE
 
 RUN ln -s /workspace /app
 

heroku-22-build/Dockerfile

@@ -1,3 +1,4 @@
-FROM robuust/heroku:22
+ARG BASE_IMAGE=robuust/heroku:22
+FROM $BASE_IMAGE
 COPY setup.sh /tmp/setup.sh
 RUN /tmp/setup.sh

heroku-22-cnb-build/Dockerfile

@@ -1,4 +1,5 @@
-FROM heroku/heroku:22-build
+ARG BASE_IMAGE=heroku/heroku:22-build
+FROM $BASE_IMAGE
 
 RUN groupadd heroku --gid 1000 && \
   useradd heroku -u 1000 -g 1000 -s /bin/bash -m
@@ -8,8 +9,10 @@ RUN mkdir /app && \
 
 ENV CNB_USER_ID=1000
 ENV CNB_GROUP_ID=1000
-
 ENV CNB_STACK_ID "heroku-22"
+ENV CNB_TARGET_OS="linux"
+ENV CNB_TARGET_ARCH="amd64"
+
 LABEL io.buildpacks.stack.id="heroku-22"
 
 USER heroku

heroku-22-cnb/Dockerfile

@@ -1,4 +1,5 @@
-FROM heroku/heroku:22
+ARG BASE_IMAGE=heroku/heroku:22
+FROM $BASE_IMAGE
 
 RUN ln -s /workspace /app
 

heroku-24-build/.dockerignore

@@ -0,0 +1 @@
+installed-packages*.txt

heroku-24-build/Dockerfile

@@ -0,0 +1,12 @@
+ARG BASE_IMAGE=heroku/heroku:24
+FROM $BASE_IMAGE
+USER root
+RUN --mount=target=/build /build/setup.sh
+
+# https://github.com/buildpacks/spec/blob/main/platform.md#build-image
+USER 1002
+ENV CNB_USER_ID=1002
+ENV CNB_GROUP_ID=1000
+ENV CNB_STACK_ID "heroku-24"
+ENV CNB_TARGET_OS=$TARGETOS
+ENV CNB_TARGET_ARCH=$TARGETARCH