feat: PAN-1832 multi arch builds

pantos-io · Aug 8, 2024 · 92ebf75 · 92ebf75
1 parent b6e9785
commit 92ebf75
Show file tree

Hide file tree

Showing 10 changed files with 27 additions and 74 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -11,11 +11,6 @@ on:
         description: 'Environment where the secrets are stored'
         required: false
         type: string
-      architecture:
-        description: 'Architecture to build'
-        required: false
-        type: string
-        default: "amd64"
     secrets:
       GPG_PRIVATE_KEY:
         description: 'GPG private key'
@@ -31,8 +26,7 @@ on:
 jobs:
   build-deb:
     name: Build and attach .deb and .whl packages
-    # TODO: Change ubuntu-20.04 for the ARM public runner
-    runs-on: ${{ inputs.architecture == 'amd64' && 'ubuntu-latest' || 'ubuntu-20.04' }}
+    runs-on: 'ubuntu-latest'
     environment: ${{ inputs.environment }}
     outputs:
       version: ${{ steps.is-signed-build.outputs.built-version }}
@@ -42,6 +36,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.10"]
+        architecture: ["amd64", "arm64"]
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
@@ -63,7 +58,7 @@ jobs:
         uses: pantos-io/ci-workflows/.github/actions/install-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-          runner-os: ${{ inputs.architecture == 'amd64' && 'ubuntu-latest' || 'ubuntu-20.04' }}
+          runner-os: 'ubuntu-latest'
 
       - name: Check secrets
         id: is-signed-build
@@ -82,28 +77,6 @@ jobs:
           gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.GPG_PASSPHRASE }}
 
-      - name: Install conda dependencies
-        run: |
-          ARCH=$(uname -m)
-          if [ "$ARCH" = "x86_64" ]; then
-              MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh";
-          elif [ "$ARCH" = "aarch64" ]; then
-              MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh";
-          else
-              echo "Unsupported architecture: $ARCH";
-              exit 1;
-          fi
-          wget "$MINICONDA_URL" -O miniconda.sh
-          bash miniconda.sh -b
-          rm -f miniconda.sh
-        shell: sh
-
-      - name: Install build dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install build-essential debhelper devscripts equivs dh-virtualenv python3-venv dh-sysuser dh-exec -y
-          sudo make debian-build-deps
-
       - name: Install signing dependencies
         if: steps.is-signed-build.outputs.HAS_SECRETS == 'true'
         run: |
@@ -117,9 +90,7 @@ jobs:
 
       - name: Build package
         run: |
-          make debian debian-full
-          make wheel
-        shell: sh
+          make docker-debian-build ARGS="--platform=linux/${{ matrix.architecture }}"
 
       - name: Sign package
         if: steps.is-signed-build.outputs.HAS_SECRETS == 'true'
@@ -129,5 +100,5 @@ jobs:
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
-          name: build-${{ inputs.architecture }}
+          name: build-${{ matrix.architecture }}
           path: dist/*
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -55,13 +55,6 @@ jobs:
           ssh-keygen -t ed25519 -f signer_key.pem -N ''
           chmod 777 signer_key.pem
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
-        id: buildx
-
       - name: Cache Docker layers
         uses: actions/cache@v3
         with:
@@ -102,10 +95,9 @@ jobs:
 
       - name: Build and load
         run: |
-          make docker-build ARGS="--set "*.cache-from=type=local,src=/tmp/.buildx-cache" \
+          make docker-build ARGS='--set "*.cache-from=type=local,src=/tmp/.buildx-cache" \
             --set "*.cache-to=type=local,dest=/tmp/.buildx-cache-new" \
-            --set "*.platform=linux/amd64" \
-            --builder ${{ steps.buildx.outputs.name }}"
+            --set "*.platform=linux/amd64"'
 
       - name: Test image
         timeout-minutes: 10
@@ -134,8 +126,6 @@ jobs:
 
   build:
     uses: ./.github/workflows/build.yml
-    with:
-      architecture: amd64
 
   install:
     needs: [build]
@@ -159,9 +149,6 @@ jobs:
            Makefile
            docker-compose.yml
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         id: buildx

diff --git a/.github/workflows/docker-vulnerabilities.yaml b/.github/workflows/docker-vulnerabilities.yaml
@@ -36,13 +36,6 @@ jobs:
                   repo.anaconda.com:443         
             
             - uses: actions/checkout@v4
-
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
-
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
-              id: buildx
 
             - name: Cache Docker layers
               uses: actions/cache@v3
@@ -58,7 +51,6 @@ jobs:
                     --set "*.cache-from=type=local,src=/tmp/.buildx-cache" \
                     --set "*.cache-to=type=local,dest=/tmp/.buildx-cache-new" \
                     --set "*.platform=linux/amd64" \
-                    --builder ${{ steps.buildx.outputs.name }} \
                     -f docker-compose.yml \
                     --load \
                     app worker

diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml
@@ -39,13 +39,6 @@ jobs:
           egress-policy: audit
       - uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
-        id: buildx
-
       - name: Cache Docker layers
         uses: actions/cache@v3
         with:
@@ -89,8 +82,7 @@ jobs:
           docker buildx bake \
             --set "*.cache-from=type=local,src=/tmp/.buildx-cache" \
             --set "*.cache-to=type=local,dest=/tmp/.buildx-cache-new" \
-            --set "*.platform=linux/amd64,linux/arm64 \
-            --builder ${{ steps.buildx.outputs.name }} \
+            --set "*.platform=linux/amd64,linux/arm64" \
             --sbom=true \
             --push \
             -f docker-compose.yml \

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -69,7 +69,6 @@ jobs:
       # We need to use a semver that doesn't start with a v as debian will remove it anyways
       version: ${{ needs.define-environment.outputs.deployment_version }}
       environment: debian-release
-      architecture: amd64
 
   add-assets:
     name: Add Assets to the ${{ github.event.release.tag_name }} Release

diff --git a/Dockerfile b/Dockerfile
@@ -28,7 +28,7 @@ COPY . /app
 
 RUN make debian-build-deps
 
-RUN make debian
+RUN make debian debian-full
 
 FROM bitnami/minideb:bookworm AS prod
 
@@ -37,10 +37,11 @@ RUN apt-get update
 # Do not copy the configurator package
 COPY --from=dev /app/dist/pantos-service-node_*.deb .
 
-RUN if [ -f ./*-signed.deb ]; then \
-    apt-get install -y --no-install-recommends ./*-signed.deb; \
+RUN ARCH=$(dpkg --print-architecture) && \
+    if [ -f ./*-${ARCH}-signed.deb ]; then \
+    apt-get install -y --no-install-recommends ./*_${ARCH}-signed.deb ./*_all-signed.deb; \
     else \
-    apt-get install -y --no-install-recommends ./*.deb; \
+    apt-get install -y --no-install-recommends ./*_${ARCH}.deb ./*_all.deb; \
     fi && \
     rm -rf *.deb && \
     apt-get clean && \

diff --git a/Makefile b/Makefile
@@ -143,10 +143,19 @@ debian:
 	fi; \
 	dpkg-buildpackage -uc -us -g
 	mkdir -p dist
-	mv ../$(debian_package) dist/
+	ARCHITECTURE=$$(dpkg --print-architecture); \
+	mv ../$(debian_package) dist/panto-service-node_$(PANTOS_SERVICE_NODE_VERSION)_$${ARCHITECTURE}.deb
 
+.PHONY: debian-all
 debian-all: debian debian-full
 
+.PHONY: docker-debian-build
+docker-debian-build:
+	docker build -t pantos-service-node-build -f Dockerfile --target dev . $(ARGS);
+	CONTAINER_ID=$$(docker create pantos-service-node-build); \
+    docker cp $${CONTAINER_ID}:/app/dist/ .; \
+    docker rm $${CONTAINER_ID}
+
 .PHONY: signer-key
 signer-key:
 	@if ! command -v ssh-keygen &> /dev/null; then \

diff --git a/configurator/DEBIAN/.gitignore b/configurator/DEBIAN/.gitignore
@@ -4,4 +4,5 @@ tmp
 *.debhelper
 *.substvars
 debhelper*
-files
+files
+control
diff --git a/debian/rules b/debian/rules
@@ -44,7 +44,7 @@ build-arch:
 
 override_dh_virtualenv:
 	. $$(conda info --base)/etc/profile.d/conda.sh && \
-	conda create -y --prefix $(POETRY_VIRTUALENVS_PATH) python=$(PYTHON_VERSION) && \
+	conda create -y -c defaults -c conda-forge --prefix $(POETRY_VIRTUALENVS_PATH) python=$(PYTHON_VERSION) && \
 	conda activate $(POETRY_VIRTUALENVS_PATH) && \
 	dh_virtualenv $(DH_VENV_ARGS)
 

diff --git a/environment.yml b/environment.yml
@@ -1,5 +1,6 @@
 name: default
 channels:
   - defaults
+  - conda-forge
 dependencies:
   - python=3.12