From 92ebf75afe11dc2913e157d91f9c890e3d594061 Mon Sep 17 00:00:00 2001 From: Joe Matthew Date: Sun, 4 Aug 2024 17:00:45 +0200 Subject: [PATCH] feat: PAN-1832 multi arch builds --- .github/workflows/build.yml | 39 +++---------------- .github/workflows/ci.yaml | 17 +------- .github/workflows/docker-vulnerabilities.yaml | 8 ---- .github/workflows/publish-docker.yaml | 10 +---- .github/workflows/release.yaml | 1 - Dockerfile | 9 +++-- Makefile | 11 +++++- configurator/DEBIAN/.gitignore | 3 +- debian/rules | 2 +- environment.yml | 1 + 10 files changed, 27 insertions(+), 74 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 35afd55..f2e103c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,11 +11,6 @@ on: description: 'Environment where the secrets are stored' required: false type: string - architecture: - description: 'Architecture to build' - required: false - type: string - default: "amd64" secrets: GPG_PRIVATE_KEY: description: 'GPG private key' @@ -31,8 +26,7 @@ on: jobs: build-deb: name: Build and attach .deb and .whl packages - # TODO: Change ubuntu-20.04 for the ARM public runner - runs-on: ${{ inputs.architecture == 'amd64' && 'ubuntu-latest' || 'ubuntu-20.04' }} + runs-on: 'ubuntu-latest' environment: ${{ inputs.environment }} outputs: version: ${{ steps.is-signed-build.outputs.built-version }} @@ -42,6 +36,7 @@ jobs: fail-fast: false matrix: python-version: ["3.10"] + architecture: ["amd64", "arm64"] steps: - name: Harden Runner uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 @@ -63,7 +58,7 @@ jobs: uses: pantos-io/ci-workflows/.github/actions/install-poetry@v1 with: python-version: ${{ matrix.python-version }} - runner-os: ${{ inputs.architecture == 'amd64' && 'ubuntu-latest' || 'ubuntu-20.04' }} + runner-os: 'ubuntu-latest' - name: Check secrets id: is-signed-build @@ -82,28 +77,6 @@ jobs: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} passphrase: ${{ secrets.GPG_PASSPHRASE }} - - name: Install conda dependencies - run: | - ARCH=$(uname -m) - if [ "$ARCH" = "x86_64" ]; then - MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh"; - elif [ "$ARCH" = "aarch64" ]; then - MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh"; - else - echo "Unsupported architecture: $ARCH"; - exit 1; - fi - wget "$MINICONDA_URL" -O miniconda.sh - bash miniconda.sh -b - rm -f miniconda.sh - shell: sh - - - name: Install build dependencies - run: | - sudo apt-get update - sudo apt-get install build-essential debhelper devscripts equivs dh-virtualenv python3-venv dh-sysuser dh-exec -y - sudo make debian-build-deps - - name: Install signing dependencies if: steps.is-signed-build.outputs.HAS_SECRETS == 'true' run: | @@ -117,9 +90,7 @@ jobs: - name: Build package run: | - make debian debian-full - make wheel - shell: sh + make docker-debian-build ARGS="--platform=linux/${{ matrix.architecture }}" - name: Sign package if: steps.is-signed-build.outputs.HAS_SECRETS == 'true' @@ -129,5 +100,5 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: build-${{ inputs.architecture }} + name: build-${{ matrix.architecture }} path: dist/* diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1533d7f..08a354b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -55,13 +55,6 @@ jobs: ssh-keygen -t ed25519 -f signer_key.pem -N '' chmod 777 signer_key.pem - - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 - id: buildx - - name: Cache Docker layers uses: actions/cache@v3 with: @@ -102,10 +95,9 @@ jobs: - name: Build and load run: | - make docker-build ARGS="--set "*.cache-from=type=local,src=/tmp/.buildx-cache" \ + make docker-build ARGS='--set "*.cache-from=type=local,src=/tmp/.buildx-cache" \ --set "*.cache-to=type=local,dest=/tmp/.buildx-cache-new" \ - --set "*.platform=linux/amd64" \ - --builder ${{ steps.buildx.outputs.name }}" + --set "*.platform=linux/amd64"' - name: Test image timeout-minutes: 10 @@ -134,8 +126,6 @@ jobs: build: uses: ./.github/workflows/build.yml - with: - architecture: amd64 install: needs: [build] @@ -159,9 +149,6 @@ jobs: Makefile docker-compose.yml - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 id: buildx diff --git a/.github/workflows/docker-vulnerabilities.yaml b/.github/workflows/docker-vulnerabilities.yaml index 5e9062e..d5451e6 100644 --- a/.github/workflows/docker-vulnerabilities.yaml +++ b/.github/workflows/docker-vulnerabilities.yaml @@ -36,13 +36,6 @@ jobs: repo.anaconda.com:443 - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 - id: buildx - name: Cache Docker layers uses: actions/cache@v3 @@ -58,7 +51,6 @@ jobs: --set "*.cache-from=type=local,src=/tmp/.buildx-cache" \ --set "*.cache-to=type=local,dest=/tmp/.buildx-cache-new" \ --set "*.platform=linux/amd64" \ - --builder ${{ steps.buildx.outputs.name }} \ -f docker-compose.yml \ --load \ app worker diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index a3c4756..d7daaf8 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -39,13 +39,6 @@ jobs: egress-policy: audit - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 - id: buildx - - name: Cache Docker layers uses: actions/cache@v3 with: @@ -89,8 +82,7 @@ jobs: docker buildx bake \ --set "*.cache-from=type=local,src=/tmp/.buildx-cache" \ --set "*.cache-to=type=local,dest=/tmp/.buildx-cache-new" \ - --set "*.platform=linux/amd64,linux/arm64 \ - --builder ${{ steps.buildx.outputs.name }} \ + --set "*.platform=linux/amd64,linux/arm64" \ --sbom=true \ --push \ -f docker-compose.yml \ diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3140144..9268cf7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -69,7 +69,6 @@ jobs: # We need to use a semver that doesn't start with a v as debian will remove it anyways version: ${{ needs.define-environment.outputs.deployment_version }} environment: debian-release - architecture: amd64 add-assets: name: Add Assets to the ${{ github.event.release.tag_name }} Release diff --git a/Dockerfile b/Dockerfile index eec73f6..3397be3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ COPY . /app RUN make debian-build-deps -RUN make debian +RUN make debian debian-full FROM bitnami/minideb:bookworm AS prod @@ -37,10 +37,11 @@ RUN apt-get update # Do not copy the configurator package COPY --from=dev /app/dist/pantos-service-node_*.deb . -RUN if [ -f ./*-signed.deb ]; then \ - apt-get install -y --no-install-recommends ./*-signed.deb; \ +RUN ARCH=$(dpkg --print-architecture) && \ + if [ -f ./*-${ARCH}-signed.deb ]; then \ + apt-get install -y --no-install-recommends ./*_${ARCH}-signed.deb ./*_all-signed.deb; \ else \ - apt-get install -y --no-install-recommends ./*.deb; \ + apt-get install -y --no-install-recommends ./*_${ARCH}.deb ./*_all.deb; \ fi && \ rm -rf *.deb && \ apt-get clean && \ diff --git a/Makefile b/Makefile index 73335e6..2e4334e 100644 --- a/Makefile +++ b/Makefile @@ -143,10 +143,19 @@ debian: fi; \ dpkg-buildpackage -uc -us -g mkdir -p dist - mv ../$(debian_package) dist/ + ARCHITECTURE=$$(dpkg --print-architecture); \ + mv ../$(debian_package) dist/panto-service-node_$(PANTOS_SERVICE_NODE_VERSION)_$${ARCHITECTURE}.deb +.PHONY: debian-all debian-all: debian debian-full +.PHONY: docker-debian-build +docker-debian-build: + docker build -t pantos-service-node-build -f Dockerfile --target dev . $(ARGS); + CONTAINER_ID=$$(docker create pantos-service-node-build); \ + docker cp $${CONTAINER_ID}:/app/dist/ .; \ + docker rm $${CONTAINER_ID} + .PHONY: signer-key signer-key: @if ! command -v ssh-keygen &> /dev/null; then \ diff --git a/configurator/DEBIAN/.gitignore b/configurator/DEBIAN/.gitignore index 947b9f7..4518a6c 100644 --- a/configurator/DEBIAN/.gitignore +++ b/configurator/DEBIAN/.gitignore @@ -4,4 +4,5 @@ tmp *.debhelper *.substvars debhelper* -files \ No newline at end of file +files +control diff --git a/debian/rules b/debian/rules index 25cf337..3512a84 100755 --- a/debian/rules +++ b/debian/rules @@ -44,7 +44,7 @@ build-arch: override_dh_virtualenv: . $$(conda info --base)/etc/profile.d/conda.sh && \ - conda create -y --prefix $(POETRY_VIRTUALENVS_PATH) python=$(PYTHON_VERSION) && \ + conda create -y -c defaults -c conda-forge --prefix $(POETRY_VIRTUALENVS_PATH) python=$(PYTHON_VERSION) && \ conda activate $(POETRY_VIRTUALENVS_PATH) && \ dh_virtualenv $(DH_VENV_ARGS) diff --git a/environment.yml b/environment.yml index 505b2f9..1d690fa 100644 --- a/environment.yml +++ b/environment.yml @@ -1,5 +1,6 @@ name: default channels: - defaults + - conda-forge dependencies: - python=3.12