installs python 3.9 from minimal base image into builder base

builder-base/Dockerfile

@@ -19,8 +19,9 @@
 
 ARG BUILDER_IMAGE
 ARG BASE_IMAGE
+ARG PYTHON_IMAGE
 ARG FINAL_STAGE_BASE
-
+ARG AL_TAG
 
 FROM ${BUILDER_IMAGE} as aws-cli
 ARG TARGETARCH
@@ -97,6 +98,8 @@ ARG TARGETARCH
 WORKDIR /workdir
 ARG PACKER_VERSION
 ENV PACKER_VERSION=$PACKER_VERSION
+ARG PACKER_ANSIBLE_PLUGIN
+ENV PACKER_ANSIBLE_PLUGIN=$PACKER_ANSIBLE_PLUGIN
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_packer.sh ./checksums/packer-${TARGETARCH}-checksum /
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
@@ -194,7 +197,29 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
     /install_bash.sh && \
     /remove_yum_packages.sh
 
-FROM ${BUILDER_IMAGE} as ansible
+# for al2 builds we pull python3.9 from the minimal image build since al2
+# does not ship 3.9
+FROM ${PYTHON_IMAGE} as python-3.9
+ARG ANSIBLE_VERSION
+ENV ANSIBLE_VERSION=$ANSIBLE_VERSION
+ARG PYWINRM_VERSION
+ENV PYWINRM_VERSION=$PYWINRM_VERSION
+RUN python3 -m ensurepip --upgrade && \
+    pip3 install --no-cache-dir ansible-core==$ANSIBLE_VERSION && \
+    pip3 install --no-cache-dir pywinrm==$PYWINRM_VERSION
+
+FROM ${BUILDER_IMAGE} as ansible-al2
+
+COPY --link --from=python-3.9 / /ansible
+# we want python2 to be the version symlinkd to python
+# we also do not want to overwrite the rpm db
+# this will leave the packages pulled in from the python image in
+# a slightly odd state in that they are installed yet not in the rpm db
+RUN set -x && \
+    unlink /ansible/usr/bin/python && \
+    rm -rf /var/lib/
+
+FROM ${BUILDER_IMAGE} as ansible-al2023
 ARG TARGETARCH
 WORKDIR /workdir
 ARG ANSIBLE_VERSION
@@ -208,6 +233,8 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
     /install_ansible.sh && \
     /remove_yum_packages.sh
 
+FROM ansible-${AL_TAG} as ansible
+
 FROM ${BUILDER_IMAGE} as golang-1.16
 ARG TARGETARCH
 ARG GOLANG_VERSION_116

builder-base/Makefile

@@ -28,7 +28,8 @@ LATEST_IMAGE=$(shell echo $(LATEST_TAGS) | sed "s/ \+/,/g")
 BASE_IMAGE_REPO?=public.ecr.aws/eks-distro-build-tooling
 BASE_IMAGE_NAME?=eks-distro-minimal-base-kind
 BASE_IMAGE?=$(BASE_IMAGE_REPO)/$(BASE_IMAGE_NAME):$(call BASE_TAG_FROM_TAG_FILE,$(BASE_IMAGE_NAME))
-
+PYTHON_IMAGE_NAME?=eks-distro-minimal-base-python
+PYTHON_IMAGE?=$(BASE_IMAGE_REPO)/$(PYTHON_IMAGE_NAME):$(call BASE_TAG_FROM_TAG_FILE,$(PYTHON_IMAGE_NAME)-3.9)
 # using the minimal-base-builder as the base for the download/gcc stages of
 # this build since it doesnt change too often, limiting churn/rebuilds
 BUILDER_IMAGE_NAME?=eks-distro-minimal-base
@@ -46,14 +47,16 @@ IGNORE_NO_CACHE?=false
 NETRC=--secret id=netrc,src=$(HOME)/.netrc
 
 define BASE_TAG_FROM_TAG_FILE
-$(shell yq e ".al$(AL_TAG).$(1)" $(MAKE_ROOT)/../EKS_DISTRO_TAG_FILE.yaml)
+$(shell yq e ".al$(AL_TAG).\"$(1)\"" $(MAKE_ROOT)/../EKS_DISTRO_TAG_FILE.yaml)
 endef
 
 define NEWLINE
 
 
 endef
 
+REALPATH=TZ=utc $(shell if [ "$$(uname -s)" = "Darwin" ] && command -v grealpath &> /dev/null; then echo grealpath; else echo realpath; fi)
+
 GOLANG_RPM_OUTPUT_DIR?=$(MAKE_ROOT)/tmp/golang-downloads
 # Resolves to the download-golang-% targets for each golang version present in ./versions.yaml
 # For example, the `GOLANG_VERSION_116: 1.16.15-3` entry in versions.yaml resolves to `download-golang-1.16.15-3`, and so on
@@ -91,10 +94,12 @@ images-%:
 		--frontend dockerfile.v0 \
 		--opt platform=$(BUILDKIT_PLATFORMS) \
 		--opt build-arg:BASE_IMAGE=$(BASE_IMAGE) \
+		--opt build-arg:PYTHON_IMAGE=$(PYTHON_IMAGE) \
 		--opt build-arg:BUILDER_IMAGE=$(BUILDER_IMAGE) \
 		--opt build-arg:GOPROXY=$(GOPROXY) \
+		--opt build-arg:AL_TAG=al$(AL_TAG) \
 		--opt build-arg:FINAL_STAGE_BASE=$(FINAL_STAGE_BASE) \
-		--opt build-arg:GOLANG_RPM_SOURCE_DIR=$(shell realpath --relative-to $(MAKE_ROOT) $(GOLANG_RPM_OUTPUT_DIR)) \
+		--opt build-arg:GOLANG_RPM_SOURCE_DIR=$(shell $(REALPATH) --relative-to $(MAKE_ROOT) $(GOLANG_RPM_OUTPUT_DIR)) \
 		$(foreach BUILD_ARG,$(IMAGE_BUILD_ARGS),--opt build-arg:$(BUILD_ARG)=$($(BUILD_ARG))) \
 		--export-cache type=inline \
 		$(foreach repo,$(IMPORT_CACHE_REPOS),--import-cache type=registry,ref=$(repo)/builder-base:$(word 1,$(LATEST))) \

builder-base/checksums/packer-amd64-checksum

@@ -1 +1 @@
-1f17a724e5ccc696010c842e6d2bb2c2749ab18ce7bf06482012d3ddb9edeef2  packer_1.8.5_linux_amd64.zip
+6cd5269c4245aa8c99e551d1b862460d63fe711c58bec618fade25f8492e80d9  packer_1.9.4_linux_amd64.zip

builder-base/checksums/packer-arm64-checksum

@@ -1 +1 @@
-28ad00415862586bd4877b6cb5db6b4340787728dcc570456b8c4fdc482ac498  packer_1.8.5_linux_arm64.zip
+f00a4fc221b20a166cfac8a63513054775988a068667517bb3edcfab8b1700ba  packer_1.9.4_linux_arm64.zip

builder-base/scripts/install_final.sh

@@ -28,18 +28,14 @@ usermod --shell /bin/bash root
 
 # user for goss/imagebuilder
 # to make sure the home dir is created correctly, tmp move the goss plugin
-# on arm goss does not exist
-if [ -f /home/imagebuilder/.packer.d/plugins/packer-provisioner-goss ]; then
-    mv /home/imagebuilder/.packer.d/plugins/packer-provisioner-goss /tmp
-fi
+mv /home/imagebuilder/.packer.d /tmp
 
 rm -rf /home/imagebuilder
 useradd -ms /bin/bash -u 1100 imagebuilder
 
-if [ -f /tmp/packer-provisioner-goss ]; then
-    mkdir -p /home/imagebuilder/.packer.d/plugins/
-    mv /tmp/packer-provisioner-goss /home/imagebuilder/.packer.d/plugins/
-fi
+mv /tmp/.packer.d /home/imagebuilder/
+
+chown -R imagebuilder:imagebuilder /home/imagebuilder
 
 # directory setup
 mkdir -p /go/src/github.com/aws/eks-distro
@@ -79,8 +75,7 @@ if [ "${FINAL_STAGE_BASE}" = "full-copy-stage" ]; then
     yum install -y \
         gcc \
         openssl-devel \
-        pkgconfig \
-        python3-pip
+        pkgconfig 
 
     # for building containerd
     yum install -y \

builder-base/scripts/install_packer.sh

@@ -33,6 +33,10 @@ function install_packer() {
         $PACKER_DOWNLOAD_URL
     sha256sum -c $BASE_DIR/packer-$TARGETARCH-checksum
     unzip -o packer_${PACKER_VERSION}_linux_$TARGETARCH.zip -d $USR_LOCAL_BIN
+
+    mkdir -p /packer/home/imagebuilder
+
+    PACKER_CONFIG_DIR=/packer/home/imagebuilder $USR_LOCAL_BIN/packer plugins install github.com/hashicorp/ansible ${PACKER_ANSIBLE_PLUGIN}
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_packer

builder-base/versions.yaml

@@ -1,5 +1,5 @@
 AMAZON_ECR_CRED_HELPER_VERSION: 0.6.0
-ANSIBLE_VERSION: 2.11.12
+ANSIBLE_VERSION: 2.15.3
 BUILDKIT_VERSION: v0.10.5
 DOCKER_BUILDX_VERSION: v0.9.1
 GITHUB_CLI_VERSION: 2.21.1
@@ -17,7 +17,8 @@ HUGO_VERSION: 0.85.0
 LINUXKIT_VERSION: v0.0.0-20220415093837-b710224cdf9a
 NODEJS_VERSION: v16.18.1
 OVERRIDE_BASH_VERSION: 4.3
-PACKER_VERSION: 1.8.5
+PACKER_VERSION: 1.9.4
+PACKER_ANSIBLE_PLUGIN: v1.1.0
 PYWINRM_VERSION: 0.4.1
 SKOPEO_VERSION: v1.5.2
 YQ_VERSION: v4.30.6