feat: PAN-1832 multi arch builds (pantos-io#73)

.github/workflows/build.yml

@@ -11,11 +11,6 @@ on:
         description: 'Environment where the secrets are stored'
         required: false
         type: string
-      architecture:
-        description: 'Architecture to build'
-        required: false
-        type: string
-        default: "amd64"
     secrets:
       GPG_PRIVATE_KEY:
         description: 'GPG private key'
@@ -31,8 +26,7 @@ on:
 jobs:
   build-deb:
     name: Build and attach .deb and .whl packages
-    # TODO: Change ubuntu-20.04 for the ARM public runner
-    runs-on: ${{ inputs.architecture == 'amd64' && 'ubuntu-latest' || 'ubuntu-20.04' }}
+    runs-on: 'ubuntu-latest'
     environment: ${{ inputs.environment }}
     outputs:
       version: ${{ steps.is-signed-build.outputs.built-version }}
@@ -42,11 +36,12 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.10"]
+        architecture: ["amd64", "arm64"]
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
         with:
-          egress-policy: block
+          egress-policy: audit
           allowed-endpoints: >
             azure.archive.ubuntu.com:80
             esm.ubuntu.com:443
@@ -56,14 +51,22 @@ jobs:
             packages.microsoft.com:443
             pypi.org:443
             repo.anaconda.com:443
+            registry-1.docker.io:443
             
       - uses: actions/checkout@v4
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        id: buildx
+
       - name: Set up Poetry
         uses: pantos-io/ci-workflows/.github/actions/install-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-          runner-os: ${{ inputs.architecture == 'amd64' && 'ubuntu-latest' || 'ubuntu-20.04' }}
+          runner-os: 'ubuntu-latest'
 
       - name: Check secrets
         id: is-signed-build
@@ -82,28 +85,6 @@ jobs:
           gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.GPG_PASSPHRASE }}
 
-      - name: Install conda dependencies
-        run: |
-          ARCH=$(uname -m)
-          if [ "$ARCH" = "x86_64" ]; then
-              MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh";
-          elif [ "$ARCH" = "aarch64" ]; then
-              MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh";
-          else
-              echo "Unsupported architecture: $ARCH";
-              exit 1;
-          fi
-          wget "$MINICONDA_URL" -O miniconda.sh
-          bash miniconda.sh -b
-          rm -f miniconda.sh
-        shell: sh
-
-      - name: Install build dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install build-essential debhelper devscripts equivs dh-virtualenv python3-venv dh-sysuser dh-exec -y
-          sudo make debian-build-deps
-
       - name: Install signing dependencies
         if: steps.is-signed-build.outputs.HAS_SECRETS == 'true'
         run: |
@@ -117,9 +98,10 @@ jobs:
 
       - name: Build package
         run: |
-          make debian debian-full
-          make wheel
-        shell: sh
+          make docker-debian-build ARGS='--platform=linux/${{ matrix.architecture }} \
+            --cache-from=type=local,src=/tmp/.buildx-cache \
+            --cache-to=type=local,dest=/tmp/.buildx-cache-new \
+            --builder ${{ steps.buildx.outputs.name }}'
 
       - name: Sign package
         if: steps.is-signed-build.outputs.HAS_SECRETS == 'true'
@@ -129,5 +111,5 @@ jobs:
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
-          name: build-${{ inputs.architecture }}
+          name: build-${{ matrix.architecture }}
           path: dist/*

.github/workflows/ci.yaml

@@ -16,12 +16,15 @@ jobs:
     runs-on: ubuntu-latest
     permissions: 
       contents: read
-
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: ['amd64', 'arm64']
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
         with:
-          egress-policy: block
+          egress-policy: audit
           allowed-endpoints: >
             api.github.com:443
             auth.docker.io:443
@@ -56,14 +59,14 @@ jobs:
           chmod 777 signer_key.pem
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+        uses: docker/setup-buildx-action@v3
         id: buildx
 
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-v1.0-service-node-${{ github.ref_name }}
@@ -102,10 +105,10 @@ jobs:
 
       - name: Build and load
         run: |
-          make docker-build ARGS="--set "*.cache-from=type=local,src=/tmp/.buildx-cache" \
+          make docker-build ARGS='--set "*.cache-from=type=local,src=/tmp/.buildx-cache" \
             --set "*.cache-to=type=local,dest=/tmp/.buildx-cache-new" \
-            --set "*.platform=linux/amd64" \
-            --builder ${{ steps.buildx.outputs.name }}"
+            --set "*.platform=linux/${{ matrix.arch }}" \
+            --builder ${{ steps.buildx.outputs.name }}'
 
       - name: Test image
         timeout-minutes: 10
@@ -134,8 +137,6 @@ jobs:
 
   build:
     uses: ./.github/workflows/build.yml
-    with:
-      architecture: amd64
 
   install:
     needs: [build]
@@ -159,9 +160,6 @@ jobs:
            Makefile
            docker-compose.yml
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         id: buildx

.github/workflows/docker-vulnerabilities.yaml

@@ -8,6 +8,10 @@ jobs:
     docker-analysis:
         name: Trivy analysis for Docker image
         runs-on: ubuntu-latest
+        strategy:
+          fail-fast: false
+          matrix:
+            arch: [amd64, arm64]
         permissions: 
           contents: read
           # for sarif
@@ -36,16 +40,16 @@ jobs:
                   repo.anaconda.com:443         
             
             - uses: actions/checkout@v4
-            
+
             - name: Set up QEMU
-              uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+              uses: docker/setup-qemu-action@v3
 
             - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+              uses: docker/setup-buildx-action@v3
               id: buildx
 
             - name: Cache Docker layers
-              uses: actions/cache@v3
+              uses: actions/cache@v4
               with:
                 path: /tmp/.buildx-cache
                 key: ${{ runner.os }}-buildx-v1.0-service-node-${{ github.ref_name }}
@@ -57,7 +61,7 @@ jobs:
                     docker buildx bake \
                     --set "*.cache-from=type=local,src=/tmp/.buildx-cache" \
                     --set "*.cache-to=type=local,dest=/tmp/.buildx-cache-new" \
-                    --set "*.platform=linux/amd64" \
+                    --set "*.platform=linux/${{ matrix.arch }}" \
                     --builder ${{ steps.buildx.outputs.name }} \
                     -f docker-compose.yml \
                     --load \
@@ -77,7 +81,7 @@ jobs:
               uses: github/codeql-action/upload-sarif@v3
               if: always()
               with:
-                category: 'service-node-app'
+                category: 'service-node-app-${{ matrix.arch }}'
                 sarif_file: 'trivy-app-results.sarif'
 
             - name: Scan vulnerabilities worker image
@@ -92,7 +96,7 @@ jobs:
               uses: github/codeql-action/upload-sarif@v3
               if: always()
               with:
-                category: 'service-node-worker'
+                category: 'service-node-worker-${{ matrix.arch }}'
                 sarif_file: 'trivy-worker-results.sarif'
 
             - name: Move cache

.github/workflows/main.yaml

@@ -10,8 +10,6 @@ jobs:
     if: github.repository == 'pantos-io/servicenode'
     uses: ./.github/workflows/build.yml
     secrets: 'inherit'
-    with:
-      architecture: amd64
 
   publish-docker:
     uses: ./.github/workflows/publish-docker.yaml

.github/workflows/publish-docker.yaml

@@ -40,14 +40,14 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+        uses: docker/setup-buildx-action@v3
         id: buildx
 
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-v1.0-service-node-${{ github.sha }}
@@ -89,7 +89,7 @@ jobs:
           docker buildx bake \
             --set "*.cache-from=type=local,src=/tmp/.buildx-cache" \
             --set "*.cache-to=type=local,dest=/tmp/.buildx-cache-new" \
-            --set "*.platform=linux/amd64,linux/arm64 \
+            --set "*.platform=linux/amd64,linux/arm64" \
             --builder ${{ steps.buildx.outputs.name }} \
             --sbom=true \
             --push \

.github/workflows/release.yaml

@@ -69,7 +69,6 @@ jobs:
       # We need to use a semver that doesn't start with a v as debian will remove it anyways
       version: ${{ needs.define-environment.outputs.deployment_version }}
       environment: debian-release
-      architecture: amd64
 
   add-assets:
     name: Add Assets to the ${{ github.event.release.tag_name }} Release

Dockerfile

@@ -28,7 +28,7 @@ COPY . /app
 
 RUN make debian-build-deps
 
-RUN make debian
+RUN make debian debian-full
 
 FROM bitnami/minideb:bookworm AS prod
 
@@ -37,10 +37,23 @@ RUN apt-get update
 # Do not copy the configurator package
 COPY --from=dev /app/dist/pantos-service-node_*.deb .
 
-RUN if [ -f ./*-signed.deb ]; then \
-    apt-get install -y --no-install-recommends ./*-signed.deb; \
-    else \
-    apt-get install -y --no-install-recommends ./*.deb; \
+RUN ARCH=$(dpkg --print-architecture) && \
+    PKGS=$(ls ./*-signed.deb 2>/dev/null || ls ./*.deb) && \
+    INSTALLED_COUNT=0 && \
+    for pkg in $PKGS; do \
+        if [ -f "$pkg" ]; then \
+            PKG_ARCH=$(dpkg-deb --field "$pkg" Architecture) && \
+            if [ "$PKG_ARCH" = "all" ] || [ "$PKG_ARCH" = "$ARCH" ]; then \
+                apt-get install -f -y --no-install-recommends "$pkg" && \
+                INSTALLED_COUNT=$((INSTALLED_COUNT + 1)); \
+            else \
+                echo "Skipping $pkg due to architecture mismatch"; \
+            fi; \
+        fi; \
+    done && \
+    if [ "$INSTALLED_COUNT" -eq 0 ]; then \
+        echo "Error: No packages were installed" >&2; \
+        exit 1; \
     fi && \
     rm -rf *.deb && \
     apt-get clean && \

Makefile

@@ -143,10 +143,19 @@ debian:
 	fi; \
 	dpkg-buildpackage -uc -us -g
 	mkdir -p dist
-	mv ../$(debian_package) dist/
+	ARCHITECTURE=$$(dpkg --print-architecture); \
+	mv ../$(debian_package) dist/pantos-service-node_$(PANTOS_SERVICE_NODE_VERSION)_$${ARCHITECTURE}.deb
 
+.PHONY: debian-all
 debian-all: debian debian-full
 
+.PHONY: docker-debian-build
+docker-debian-build:
+	docker buildx build -t pantos-service-node-build -f Dockerfile --target dev . --load $(ARGS);
+	CONTAINER_ID=$$(docker create pantos-service-node-build); \
+    docker cp $${CONTAINER_ID}:/app/dist/ .; \
+    docker rm $${CONTAINER_ID}
+
 .PHONY: signer-key
 signer-key:
 	@if ! command -v ssh-keygen &> /dev/null; then \

configurator/DEBIAN/.gitignore

@@ -4,4 +4,5 @@ tmp
 *.debhelper
 *.substvars
 debhelper*
-files
+files
+control

debian/rules

@@ -44,7 +44,7 @@ build-arch:
 
 override_dh_virtualenv:
 	. $$(conda info --base)/etc/profile.d/conda.sh && \
-	conda create -y --prefix $(POETRY_VIRTUALENVS_PATH) python=$(PYTHON_VERSION) && \
+	conda create -y -c defaults -c conda-forge --prefix $(POETRY_VIRTUALENVS_PATH) python=$(PYTHON_VERSION) && \
 	conda activate $(POETRY_VIRTUALENVS_PATH) && \
 	dh_virtualenv $(DH_VENV_ARGS)
 

environment.yml

@@ -1,5 +1,6 @@
 name: default
 channels:
   - defaults
+  - conda-forge
 dependencies:
   - python=3.12