adds upx compression to our biggest bins in builder-base

builder-base/checksums/upx-amd64-checksum

@@ -0,0 +1 @@
+da00dabf32f2bff14f8bfc486f18e8a84b48b4eb321e109d192558f41b7a19de  upx-4.2.0-amd64_linux.tar.xz

builder-base/checksums/upx-arm64-checksum

@@ -0,0 +1 @@
+1d25c9c5c65223e815422e63278dc1cca0671d3ba2c8ea8bf2e9c1301b47c2c1  upx-4.2.0-arm64_linux.tar.xz

builder-base/scripts/install_ansible.sh

@@ -47,7 +47,15 @@ function instal_ansible() {
         cp -rf /usr/include/python3.9 ${NEWROOT}/usr/include
         cp /usr/lib/pkgconfig/python-3.9*.pc ${NEWROOT}/usr/lib/pkgconfig
         cp -rf /usr/lib/python3.9 ${NEWROOT}/usr/lib
-        cp --preserve=links /usr/lib/libpython3* ${NEWROOT}/usr/lib
+        ls -al /usr/lib/libpython3*
+        # for some reason the /usr/lib/libpython3.9.so file should be a symlink libpython3.9.so.1.0
+        # but this got lost somewhere, probably during minimal image build, so its a full copy
+        # manually recreating the symlink to avoid having a duplicated file
+        cp /usr/lib/libpython3.so /usr/lib/libpython3.9.so.1.0 ${NEWROOT}/usr/lib
+        ln -s ./libpython3.9.so.1.0 ${NEWROOT}/usr/lib/libpython3.9.so
+
+        # the static lib of python is not needed in our use case and it is quite big
+        rm ${NEWROOT}/usr/lib/python3.9/config-3.9-*-linux-gnu/libpython3.9.a
     fi
 
     chmod 755 ${NEWROOT}/usr/lib/python3.9/site-packages 

builder-base/scripts/install_base_yum_packages.sh

@@ -40,7 +40,8 @@ yum install --setopt=install_weak_deps=False -y \
     gzip \
     tar \
     unzip \
-    wget
+    wget \
+    xz
 
 chmod -R 777 /newroot
 rm -rf /newroot

builder-base/scripts/install_buildkit.sh

@@ -41,6 +41,8 @@ function install_buildkit() {
     sha256sum -c $BASE_DIR/buildkit-$TARGETARCH-checksum
     tar -C $USR -xzf buildkit-$BUILDKIT_VERSION.linux-$TARGETARCH.tar.gz
     rm -rf buildkit-$BUILDKIT_VERSION.linux-$TARGETARCH.tar.gz
+
+    time upx --best --no-lzma $USR_BIN/{buildkit-runc,buildctl,buildkitd}
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_buildkit

builder-base/scripts/install_final.sh

@@ -55,6 +55,8 @@ yum install -y \
     which \
     yum-utils
 
+time upx --best --no-lzma /usr/libexec/docker/cli-plugins/docker-buildx /usr/bin/{containerd,ctr,docker,dockerd}
+
 # We see issues in fargate when installing on top of these images
 # including this plugin appears to fix it
 # ref: https://unix.stackexchange.com/questions/348941/rpmdb-checksum-is-invalid-trying-to-install-gcc-in-a-centos-7-2-docker-image    

builder-base/scripts/install_gh_cli.sh

@@ -31,6 +31,8 @@ function install_gh_cli() {
     tar -xzf gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH.tar.gz
     mv gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH/bin/gh $USR_BIN
     rm -rf gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH.tar.gz gh_${GITHUB_CLI_VERSION}_linux_$TARGETARCH
+
+    time upx --best --no-lzma $USR_BIN/gh
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_gh_cli

builder-base/scripts/install_go_licenses.sh

@@ -30,21 +30,18 @@ function install_go_licenses() {
     # installing go-licenses has to happen after we have set the main go
     # to symlink to the one in /root/sdk to ensure go-licenses gets built
     # with GOROOT pointed to /root/sdk/go... instead of /usr/local/go so it
-    # is able to properly packages from the standard Go library
-    # We currently  use 1.19, 1.17 or 1.16, so installing for all
-    if [ "${GOLANG_MAJOR_VERSION}" = "go1.16" ]; then
-        GO111MODULE=on GOBIN=${NEWROOT}/${GOPATH}/${GOLANG_MAJOR_VERSION}/bin go install github.com/jaxesn/go-licenses@4497a2a38565e4e6ad095ea8117c25ecd622d0cc
-    else
-        GO111MODULE=on GOBIN=${NEWROOT}/${GOPATH}/${GOLANG_MAJOR_VERSION}/bin go install github.com/jaxesn/go-licenses@6800d77c11d0ef8628e7eda908b1d1149383ca48
-    fi
-
+    # is able to properly packages from the standard Go library    
+    CGO_ENABLED=0 GO111MODULE=on GOBIN=${NEWROOT}/${GOPATH}/${GOLANG_MAJOR_VERSION}/bin go install github.com/jaxesn/go-licenses@6800d77c11d0ef8628e7eda908b1d1149383ca48
+
     # symlink to go/bin and depending on which go-licenses vs is added last to
     # the final image, will take precedent and be the default
     # similiar to the strategy with golang
     mkdir -p ${NEWROOT}/${GOPATH}/bin
     ln -s ${GOPATH}/${GOLANG_MAJOR_VERSION}/bin/go-licenses ${NEWROOT}/${GOPATH}/bin/go-licenses
 
     rm -rf ${GOPATH}
+
+    time upx --best --no-lzma ${NEWROOT}/${GOPATH}/${GOLANG_MAJOR_VERSION}/bin/go-licenses
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_go_licenses

builder-base/scripts/install_golang.sh

@@ -79,6 +79,10 @@ function build::go::extract() {
     version=$(echo "$golang_version" | grep -o "[0-9].*")
     ln -s /root/sdk/go${version}/bin/go ${GOPATH}/bin/$golang_version
 
+    # newer versions of golang stopped shipping the compiled .a lib files
+    # removing from old versions since we do not need these during our builds
+    find /root/sdk/go${version}/pkg -type f -name "*.a" -delete
+
     rm -rf /tmp/go-extracted /tmp/golang-*.rpm
 }
 
@@ -93,3 +97,5 @@ done
 mkdir -p ${NEWROOT}/root
 mv /root/sdk ${NEWROOT}/root
 mv ${GOPATH} ${NEWROOT}/${GOPATH}
+
+time upx --best --no-lzma ${NEWROOT}/root/sdk/go${VERSION%-*}/bin/go ${NEWROOT}/root/sdk/go${VERSION%-*}/pkg/tool/linux_$TARGETARCH/{addr2line,asm,cgo,compile,cover,doc,link,objdump,pprof,trace,vet}

builder-base/scripts/install_goss.sh

@@ -35,6 +35,8 @@ function install_goss() {
     sha256sum -c $BASE_DIR/goss-$TARGETARCH-checksum
     tar -C ${NEWROOT}/home/imagebuilder/.packer.d/plugins -xzf packer-provisioner-goss-v${GOSS_VERSION}-linux-$TARGETARCH.tar.gz
     rm -rf packer-provisioner-goss-v${GOSS_VERSION}-linux-$TARGETARCH.tar.gz
+
+    time upx --best --no-lzma ${NEWROOT}/home/imagebuilder/.packer.d/plugins/packer-provisioner-goss
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_goss

builder-base/scripts/install_govc.sh

@@ -41,6 +41,8 @@ function install_govc() {
     tar -xf govc_Linux_$ARCH.tar.gz
     mv govc $USR_BIN/govc
     chmod +x $USR_BIN/govc
+
+    time upx --best --no-lzma $USR_BIN/govc
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_govc

builder-base/scripts/install_helm.sh

@@ -36,6 +36,8 @@ function install_helm() {
     mv linux-$TARGETARCH/helm $USR_BIN/helm
     chmod +x $USR_BIN/helm
     rm -f helm-v${HELM_VERSION}-linux-$TARGETARCH.tar.gz
+
+    time upx --best --no-lzma $USR_BIN/helm
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_helm

builder-base/scripts/install_linuxkit.sh

@@ -26,11 +26,13 @@ source $SCRIPT_ROOT/common_vars.sh
 function install_linuxkit() {
     # linuxkit is used by tinkerbell/hook for building an operating system installation environment (osie)
     # We need a higher version of linuxkit hence we do go install of a particular commit
-    GO111MODULE=on GOBIN=${GOPATH}/go1.19/bin ${GOPATH}/go1.19/bin/go install github.com/linuxkit/linuxkit/src/cmd/linuxkit@$LINUXKIT_VERSION
+    CGO_ENABLED=0  GO111MODULE=on GOBIN=${GOPATH}/go1.19/bin ${GOPATH}/go1.19/bin/go install github.com/linuxkit/linuxkit/src/cmd/linuxkit@$LINUXKIT_VERSION
 
     mv ${GOPATH}/go1.19/bin/linuxkit ${USR_BIN}/linuxkit
 
     rm -rf ${GOPATH}
+
+    time upx --best --no-lzma ${USR_BIN}/linuxkit
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_linuxkit

builder-base/scripts/install_nodejs.sh

@@ -51,6 +51,8 @@ function install_generate_attribution() {
 
     ln -s /$USR_BIN/node /usr/bin/node
     $USR_BIN/npm install
+
+    time upx --best --no-lzma $USR_BIN/node
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || (install_nodejs && install_generate_attribution)

builder-base/scripts/install_packer.sh

@@ -37,6 +37,8 @@ function install_packer() {
     mkdir -p /packer/home/imagebuilder
 
     PACKER_CONFIG_DIR=/packer/home/imagebuilder $USR_LOCAL_BIN/packer plugins install github.com/hashicorp/ansible ${PACKER_ANSIBLE_PLUGIN}
+
+    time upx --best --no-lzma $USR_LOCAL_BIN/packer /packer/home/imagebuilder/.packer.d/plugins/github.com/hashicorp/ansible/packer-plugin-ansible*_linux_${TARGETARCH}
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_packer

builder-base/scripts/install_skopeo.sh

@@ -38,6 +38,8 @@ function install_skopeo() {
 
     cd ..
     rm -rf ${GOPATH} skopeo
+
+    time upx --best --no-lzma $USR_BIN/skopeo
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_skopeo

builder-base/scripts/install_tuftool.sh

@@ -37,6 +37,8 @@ function install_tuftool() {
     cp $CARGO_HOME/bin/tuftool $USR_BIN/tuftool
 
     rm -rf $RUSTUP_HOME $CARGO_HOME
+
+    time upx --best --no-lzma $USR_BIN/tuftool
 }
 
 [ ${SKIP_INSTALL:-false} != false ] || install_tuftool

builder-base/scripts/install_upx.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -o pipefail
+
+SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+
+NEWROOT=${UPX_NEWROOT:-/}
+
+source $SCRIPT_ROOT/common_vars.sh
+
+UPX_DOWNLOAD_URL="https://github.com/upx/upx/releases/download/v${UPX_VERSION}/upx-${UPX_VERSION}-${TARGETARCH}_linux.tar.xz"
+
+function install_upx() {
+    wget --progress dot:giga $UPX_DOWNLOAD_URL
+    sha256sum -c $BASE_DIR/upx-$TARGETARCH-checksum
+    tar -xf upx-${UPX_VERSION}-${TARGETARCH}_linux.tar.xz
+    mv upx-${UPX_VERSION}-${TARGETARCH}_linux/upx ${NEWROOT}/usr/local/bin
+    rm -rf upx-${UPX_VERSION}-${TARGETARCH}_linux.tar.xz upx-${UPX_VERSION}-${TARGETARCH}_linux/upx
+}
+
+[ ${SKIP_INSTALL:-false} != false ] || install_upx

builder-base/scripts/update_shasums.sh

@@ -66,6 +66,10 @@ for TARGETARCH in arm64 amd64; do
 
     # GOSS
     echo "$(curl -sSL --retry 5 -v --silent $GOSS_CHECKSUM_URL 2>&1 | grep packer-provisioner-goss-v${GOSS_VERSION}-linux-$TARGETARCH.tar.gz | cut -d ":" -f 2)" > $CHECKSUMS_ROOT/checksums/goss-$TARGETARCH-checksum
+
+    # UPX
+    sha256=$(curl -sSL --retry 5 $UPX_DOWNLOAD_URL | sha256sum | awk '{print $1}')
+    echo "$sha256  upx-${UPX_VERSION}-${TARGETARCH}_linux.tar.xz" > $CHECKSUMS_ROOT/checksums/upx-$TARGETARCH-checksum
 done
 
 # HUGO

builder-base/scripts/versions.sh

@@ -29,3 +29,4 @@ source $SCRIPT_ROOT/install_goss.sh
 source $SCRIPT_ROOT/install_govc.sh
 source $SCRIPT_ROOT/install_hugo.sh
 source $SCRIPT_ROOT/install_bash.sh
+source $SCRIPT_ROOT/install_upx.sh

builder-base/versions.yaml

@@ -21,4 +21,5 @@ PACKER_ANSIBLE_PLUGIN: v1.1.0
 PYWINRM_VERSION: 0.4.1
 SKOPEO_VERSION: v1.5.2
 SKOPEO_VERSION_AL23: v1.13.3
+UPX_VERSION: 4.2.0
 YQ_VERSION: v4.30.6