adds upx compression to our biggest bins in builder-base

builder-base/Dockerfile

@@ -23,6 +23,19 @@ ARG PYTHON_IMAGE
 ARG FINAL_STAGE_BASE
 ARG AL_TAG
 
+FROM ${BUILDER_IMAGE} as upx
+ARG TARGETARCH
+ARG UPX_VERSION
+ENV UPX_VERSION=$UPX_VERSION
+
+WORKDIR /workdir
+COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh ./scripts/install_upx.sh \
+    ./checksums/upx-${TARGETARCH}-checksum /
+RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
+    /install_base_yum_packages.sh && \
+    /install_upx.sh && \    
+    /remove_yum_packages.sh
+
 FROM ${BUILDER_IMAGE} as aws-cli
 ARG TARGETARCH
 WORKDIR /workdir
@@ -38,6 +51,8 @@ ARG TARGETARCH
 WORKDIR /workdir
 ARG BUILDKIT_VERSION
 ENV BUILDKIT_VERSION=$BUILDKIT_VERSION
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_buildkit.sh ./checksums/buildkit-${TARGETARCH}-checksum /
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
@@ -62,6 +77,8 @@ ARG TARGETARCH
 WORKDIR /workdir
 ARG GITHUB_CLI_VERSION
 ENV GITHUB_CLI_VERSION=$GITHUB_CLI_VERSION
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_gh_cli.sh ./checksums/github-cli-${TARGETARCH}-checksum /
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
@@ -88,6 +105,8 @@ ARG PACKER_VERSION
 ENV PACKER_VERSION=$PACKER_VERSION
 ARG PACKER_ANSIBLE_PLUGIN
 ENV PACKER_ANSIBLE_PLUGIN=$PACKER_ANSIBLE_PLUGIN
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_packer.sh ./checksums/packer-${TARGETARCH}-checksum /
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
@@ -100,6 +119,8 @@ ARG TARGETARCH
 WORKDIR /workdir
 ARG HELM_VERSION
 ENV HELM_VERSION=$HELM_VERSION
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_helm.sh ./checksums/helm-${TARGETARCH}-checksum /
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
@@ -112,6 +133,8 @@ ARG TARGETARCH
 WORKDIR /workdir
 ARG NODEJS_VERSION
 ENV NODEJS_VERSION=$NODEJS_VERSION
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_nodejs.sh \
     generate-attribution/package*.json  \
@@ -141,6 +164,8 @@ ARG TARGETARCH
 WORKDIR /workdir
 ARG GOVC_VERSION
 ENV GOVC_VERSION=$GOVC_VERSION
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_govc.sh ./checksums/govc-${TARGETARCH}-checksum /
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
@@ -153,6 +178,8 @@ ARG TARGETARCH
 WORKDIR /workdir
 ARG GOSS_VERSION
 ENV GOSS_VERSION=$GOSS_VERSION
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_goss.sh ./checksums/goss-${TARGETARCH}-checksum /
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
@@ -166,6 +193,8 @@ WORKDIR /workdir
 ENV CARGO_HOME /root/.cargo
 ENV RUSTUP_HOME /root/.rustup
 ENV PATH="/root/.cargo/bin:$PATH"
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/install_gcc_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_tuftool.sh /
 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked \
@@ -211,6 +240,8 @@ ARG GOLANG_RPM_SOURCE_DIR
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_golang.sh /
 COPY $GOLANG_RPM_SOURCE_DIR/x86_64/golang*1.17*.rpm /tmp/x86_64/
@@ -227,6 +258,8 @@ ARG GOLANG_RPM_SOURCE_DIR
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_golang.sh /
 COPY $GOLANG_RPM_SOURCE_DIR/x86_64/golang*1.18*.rpm /tmp/x86_64/
@@ -243,6 +276,8 @@ ARG GOLANG_RPM_SOURCE_DIR
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_golang.sh /
 COPY $GOLANG_RPM_SOURCE_DIR/x86_64/golang*1.19*.rpm /tmp/x86_64/
@@ -259,6 +294,8 @@ ARG GOLANG_RPM_SOURCE_DIR
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_golang.sh /
 COPY $GOLANG_RPM_SOURCE_DIR/x86_64/golang*1.20*.rpm /tmp/x86_64/
@@ -275,6 +312,8 @@ ARG GOLANG_RPM_SOURCE_DIR
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+
+COPY --link --from=upx  /upx /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_golang.sh /
 COPY $GOLANG_RPM_SOURCE_DIR/x86_64/golang*1.21*.rpm /tmp/x86_64/
@@ -294,6 +333,8 @@ ENV SKOPEO_VERSION_AL23=$SKOPEO_VERSION_AL23
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+
+COPY --link --from=upx  /upx /
 COPY --link --from=golang-1.20 /golang-1.20 /
 COPY ./scripts/install_base_yum_packages.sh ./scripts/remove_yum_packages.sh ./scripts/install_gcc_yum_packages.sh ./scripts/common_vars.sh \
     ./scripts/install_skopeo.sh /
@@ -312,11 +353,13 @@ ENV LINUXKIT_VERSION=$LINUXKIT_VERSION
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+
+COPY --link --from=upx  /upx /
 COPY --link --from=golang-1.19 /golang-1.19 /
 COPY ./scripts/common_vars.sh \
     ./scripts/install_linuxkit.sh /
 RUN --mount=type=cache,target=/root/.cache/go-build,sharing=locked \
-    --mount=type=secret,id=netrc,target=/root/.netrc \    
+    --mount=type=secret,id=netrc,target=/root/.netrc \
     /install_linuxkit.sh
 
 FROM ${BUILDER_IMAGE} as go-licenses-1.17
@@ -327,6 +370,8 @@ ENV GO_LICENSES_VERSION=$GO_LICENSES_VERSION
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+
+COPY --link --from=upx  /upx /
 COPY --link --from=golang-1.17 /golang-1.17 /
 COPY ./scripts/common_vars.sh \
     ./scripts/install_go_licenses.sh /
@@ -342,6 +387,7 @@ ENV GO_LICENSES_VERSION=$GO_LICENSES_VERSION
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+COPY --link --from=upx  /upx /
 COPY --link --from=golang-1.18 /golang-1.18 /
 COPY ./scripts/common_vars.sh \
     ./scripts/install_go_licenses.sh /
@@ -357,6 +403,7 @@ ENV GO_LICENSES_VERSION=$GO_LICENSES_VERSION
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+COPY --link --from=upx  /upx /
 COPY --link --from=golang-1.19 /golang-1.19 /
 COPY ./scripts/common_vars.sh \
     ./scripts/install_go_licenses.sh /
@@ -372,6 +419,7 @@ ENV GO_LICENSES_VERSION=$GO_LICENSES_VERSION
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+COPY --link --from=upx  /upx /
 COPY --link --from=golang-1.20 /golang-1.20 /
 COPY ./scripts/common_vars.sh \
     ./scripts/install_go_licenses.sh /
@@ -387,6 +435,7 @@ ENV GO_LICENSES_VERSION=$GO_LICENSES_VERSION
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+COPY --link --from=upx  /upx /
 COPY --link --from=golang-1.21 /golang-1.21 /
 COPY ./scripts/common_vars.sh \
     ./scripts/install_go_licenses.sh /
@@ -402,6 +451,7 @@ ENV GO_VULN_CHECK_VERSION=$GO_VULN_CHECK_VERSION
 WORKDIR /workdir
 ENV GOPATH /go
 ENV PATH="/go/bin/:$PATH"
+COPY --link --from=upx  /upx /
 COPY --link --from=golang-1.21 /golang-1.21 /
 COPY ./scripts/common_vars.sh \
     ./scripts/install_go_vuln_check.sh /
@@ -423,6 +473,7 @@ COPY --link --from=helm /helm /
 COPY --link --from=tuftool /tuftool /
 COPY --link --from=buildkit /buildkit /
 COPY --link --from=skopeo /skopeo /
+COPY --link --from=upx  /upx /
 
 FROM minimal-copy-stage as full-copy-stage
 

builder-base/checksums/upx-amd64-checksum

@@ -0,0 +1 @@
+da00dabf32f2bff14f8bfc486f18e8a84b48b4eb321e109d192558f41b7a19de  upx-4.2.0-amd64_linux.tar.xz

builder-base/checksums/upx-arm64-checksum

@@ -0,0 +1 @@
+1d25c9c5c65223e815422e63278dc1cca0671d3ba2c8ea8bf2e9c1301b47c2c1  upx-4.2.0-arm64_linux.tar.xz

builder-base/scripts/install_ansible.sh

@@ -47,7 +47,16 @@ function instal_ansible() {
         cp -rf /usr/include/python3.9 ${NEWROOT}/usr/include
         cp /usr/lib/pkgconfig/python-3.9*.pc ${NEWROOT}/usr/lib/pkgconfig
         cp -rf /usr/lib/python3.9 ${NEWROOT}/usr/lib
-        cp --preserve=links /usr/lib/libpython3* ${NEWROOT}/usr/lib
+
+        # for some reason the /usr/lib/libpython3.9.so file should be a symlink libpython3.9.so.1.0
+        # but this got lost somewhere, probably during minimal image build, so its a full copy
+        # manually recreating the symlink to avoid having a duplicated file
+        cp /usr/lib/libpython3.so /usr/lib/libpython3.9.so.1.0 ${NEWROOT}/usr/lib
+        ln -s ./libpython3.9.so.1.0 ${NEWROOT}/usr/lib/libpython3.9.so
+
+        # the static lib of python is not needed in our use case and it is quite big
+        # this file only exists on the arm build for some reason
+        rm -f ${NEWROOT}/usr/lib/python3.9/config-3.9-*-linux-gnu/libpython3.9.a
     fi
 
     chmod 755 ${NEWROOT}/usr/lib/python3.9/site-packages 

builder-base/scripts/install_base_yum_packages.sh

@@ -40,7 +40,8 @@ yum install --setopt=install_weak_deps=False -y \
     gzip \
     tar \
     unzip \
-    wget
+    wget \
+    xz
 
 chmod -R 777 /newroot
 rm -rf /newroot

builder-base/scripts/install_buildkit.sh

@@ -41,6 +41,8 @@ function install_buildkit() {
     sha256sum -c $BASE_DIR/buildkit-$TARGETARCH-checksum
     tar -C $USR -xzf buildkit-$BUILDKIT_VERSION.linux-$TARGETARCH.tar.gz
     rm -rf buildkit-$BUILDKIT_VERSION.linux-$TARGETARCH.tar.gz
+
+    time upx --best --no-lzma $USR_BIN/{buildkit-runc,buildctl,buildkitd}
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_buildkit

builder-base/scripts/install_final.sh

@@ -55,6 +55,8 @@ yum install -y \
     which \
     yum-utils
 
+time upx --best --no-lzma /usr/libexec/docker/cli-plugins/docker-buildx /usr/bin/{containerd,ctr,docker,dockerd}
+
 # We see issues in fargate when installing on top of these images
 # including this plugin appears to fix it
 # ref: https://unix.stackexchange.com/questions/348941/rpmdb-checksum-is-invalid-trying-to-install-gcc-in-a-centos-7-2-docker-image    

builder-base/scripts/install_gh_cli.sh

@@ -31,6 +31,8 @@ function install_gh_cli() {
     tar -xzf gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH.tar.gz
     mv gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH/bin/gh $USR_BIN
     rm -rf gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH.tar.gz gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH
+
+    time upx --best --no-lzma $USR_BIN/gh
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_gh_cli

builder-base/scripts/install_go_licenses.sh

@@ -30,21 +30,18 @@ function install_go_licenses() {
     # installing go-licenses has to happen after we have set the main go
     # to symlink to the one in /root/sdk to ensure go-licenses gets built
     # with GOROOT pointed to /root/sdk/go... instead of /usr/local/go so it
-    # is able to properly packages from the standard Go library
-    # We currently  use 1.19, 1.17 or 1.16, so installing for all
-    if [ "${GOLANG_MAJOR_VERSION}" = "go1.16" ]; then
-        GO111MODULE=on GOBIN=${NEWROOT}/${GOPATH}/${GOLANG_MAJOR_VERSION}/bin go install github.com/jaxesn/go-licenses@4497a2a38565e4e6ad095ea8117c25ecd622d0cc
-    else
-        GO111MODULE=on GOBIN=${NEWROOT}/${GOPATH}/${GOLANG_MAJOR_VERSION}/bin go install github.com/jaxesn/go-licenses@6800d77c11d0ef8628e7eda908b1d1149383ca48
-    fi
-
+    # is able to properly packages from the standard Go library    
+    CGO_ENABLED=0 GO111MODULE=on GOBIN=${NEWROOT}/${GOPATH}/${GOLANG_MAJOR_VERSION}/bin go install github.com/jaxesn/go-licenses@6800d77c11d0ef8628e7eda908b1d1149383ca48
+
     # symlink to go/bin and depending on which go-licenses vs is added last to
     # the final image, will take precedent and be the default
     # similiar to the strategy with golang
     mkdir -p ${NEWROOT}/${GOPATH}/bin
     ln -s ${GOPATH}/${GOLANG_MAJOR_VERSION}/bin/go-licenses ${NEWROOT}/${GOPATH}/bin/go-licenses
 
     rm -rf ${GOPATH}
+
+    time upx --best --no-lzma ${NEWROOT}/${GOPATH}/${GOLANG_MAJOR_VERSION}/bin/go-licenses
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_go_licenses

builder-base/scripts/install_go_vuln_check.sh

@@ -26,6 +26,8 @@ function install_go_vuln_check() {
     GO111MODULE=on GOBIN=${NEWROOT}/${GOPATH}/${GOLANG_MAJOR_VERSION}/bin go install golang.org/x/vuln/cmd/govulncheck@$GO_VULN_CHECK_VERSION
 
     rm -rf ${GOPATH}
+
+    time upx --best --no-lzma ${NEWROOT}/${GOPATH}/${GOLANG_MAJOR_VERSION}/bin/govulncheck
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_go_vuln_check

builder-base/scripts/install_golang.sh

@@ -79,6 +79,10 @@ function build::go::extract() {
     version=$(echo "$golang_version" | grep -o "[0-9].*")
     ln -s /root/sdk/go${version}/bin/go ${GOPATH}/bin/$golang_version
 
+    # newer versions of golang stopped shipping the compiled .a lib files
+    # removing from old versions since we do not need these during our builds
+    find /root/sdk/go${version}/pkg -type f -name "*.a" -delete
+
     rm -rf /tmp/go-extracted /tmp/golang-*.rpm
 }
 
@@ -93,3 +97,5 @@ done
 mkdir -p ${NEWROOT}/root
 mv /root/sdk ${NEWROOT}/root
 mv ${GOPATH} ${NEWROOT}/${GOPATH}
+
+time upx --best --no-lzma ${NEWROOT}/root/sdk/go${VERSION%-*}/bin/go ${NEWROOT}/root/sdk/go${VERSION%-*}/pkg/tool/linux_$TARGETARCH/{addr2line,asm,cgo,compile,cover,doc,link,objdump,pprof,trace,vet}

builder-base/scripts/install_goss.sh

@@ -35,6 +35,8 @@ function install_goss() {
     sha256sum -c $BASE_DIR/goss-$TARGETARCH-checksum
     tar -C ${NEWROOT}/home/imagebuilder/.packer.d/plugins -xzf packer-provisioner-goss-v${GOSS_VERSION}-linux-$TARGETARCH.tar.gz
     rm -rf packer-provisioner-goss-v${GOSS_VERSION}-linux-$TARGETARCH.tar.gz
+
+    time upx --best --no-lzma ${NEWROOT}/home/imagebuilder/.packer.d/plugins/packer-provisioner-goss
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_goss

builder-base/scripts/install_govc.sh

@@ -41,6 +41,8 @@ function install_govc() {
     tar -xf govc_Linux_$ARCH.tar.gz
     mv govc $USR_BIN/govc
     chmod +x $USR_BIN/govc
+
+    time upx --best --no-lzma $USR_BIN/govc
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_govc

builder-base/scripts/install_helm.sh

@@ -36,6 +36,8 @@ function install_helm() {
     mv linux-$TARGETARCH/helm $USR_BIN/helm
     chmod +x $USR_BIN/helm
     rm -f helm-v${HELM_VERSION}-linux-$TARGETARCH.tar.gz
+
+    time upx --best --no-lzma $USR_BIN/helm
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_helm

builder-base/scripts/install_linuxkit.sh

@@ -26,11 +26,13 @@ source $SCRIPT_ROOT/common_vars.sh
 function install_linuxkit() {
     # linuxkit is used by tinkerbell/hook for building an operating system installation environment (osie)
     # We need a higher version of linuxkit hence we do go install of a particular commit
-    GO111MODULE=on GOBIN=${GOPATH}/go1.19/bin ${GOPATH}/go1.19/bin/go install github.com/linuxkit/linuxkit/src/cmd/linuxkit@$LINUXKIT_VERSION
+    CGO_ENABLED=0  GO111MODULE=on GOBIN=${GOPATH}/go1.19/bin ${GOPATH}/go1.19/bin/go install github.com/linuxkit/linuxkit/src/cmd/linuxkit@$LINUXKIT_VERSION
 
     mv ${GOPATH}/go1.19/bin/linuxkit ${USR_BIN}/linuxkit
 
     rm -rf ${GOPATH}
+
+    time upx --best --no-lzma ${USR_BIN}/linuxkit
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_linuxkit

builder-base/scripts/install_nodejs.sh

@@ -51,6 +51,8 @@ function install_generate_attribution() {
 
     ln -s /$USR_BIN/node /usr/bin/node
     $USR_BIN/npm install
+
+    time upx --best --no-lzma $USR_BIN/node
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || (install_nodejs && install_generate_attribution)

builder-base/scripts/install_packer.sh

@@ -37,6 +37,8 @@ function install_packer() {
     mkdir -p /packer/home/imagebuilder
 
     PACKER_CONFIG_DIR=/packer/home/imagebuilder $USR_LOCAL_BIN/packer plugins install github.com/hashicorp/ansible ${PACKER_ANSIBLE_PLUGIN}
+
+    time upx --best --no-lzma $USR_LOCAL_BIN/packer /packer/home/imagebuilder/.packer.d/plugins/github.com/hashicorp/ansible/packer-plugin-ansible*_linux_${TARGETARCH}
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_packer

builder-base/scripts/install_skopeo.sh

@@ -38,6 +38,8 @@ function install_skopeo() {
 
     cd ..
     rm -rf ${GOPATH} skopeo
+
+    time upx --best --no-lzma $USR_BIN/skopeo
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_skopeo