RWIP: docker workflow refactor GHA

.github/workflows/ci.yml

@@ -13,48 +13,84 @@ concurrency:
 
 jobs:
 
-    test-package:
+    #test-package:
 
 
-        strategy:
-            matrix:
-                tag: [latest]
-                python-version: ['3.9', '3.10']
-                aiida-core-version: [2.3, 2.5]
-            fail-fast: false
+    #    strategy:
+    #        matrix:
+    #            tag: [latest]
+    #            python-version: ['3.9', '3.10']
+    #            aiida-core-version: [2.3, 2.5]
+    #        fail-fast: false
+
+    #    runs-on: ubuntu-latest
+    #    timeout-minutes: 30
+
+    #    services:
+    #        rabbitmq:
+    #            image: rabbitmq:latest
+    #            ports:
+    #                - 5672:5672
+
+    #    steps:
+
+    #        - name: Check out app
+    #          uses: actions/checkout@v3
+
+    #        - name: Set up Python
+    #          uses: actions/setup-python@v4
+    #          with:
+    #              python-version: ${{ matrix.python-version }}
+    #              cache: pip
+    #              cache-dependency-path: |
+    #                  **/setup.cfg
+    #                  **/pyproject.toml
+    #                  **/requirements*.txt
+    #        - name: Install package
+    #          run: pip install -e .[dev] aiida-core==${{ matrix.aiida-core-version }}
+
+    #        - name: Run pytest
+    #          run: pytest -v tests --cov
+    #          env:
+    #              TAG: ${{ matrix.tag }}
+
+    #        - name: Upload coverage reports to Codecov
+    #          uses: codecov/codecov-action@v3
+    #          with:
+    #              flags: python-${{ matrix.python-version }}
+
+    build-debug:
 
         runs-on: ubuntu-latest
         timeout-minutes: 30
 
-        services:
-            rabbitmq:
-                image: rabbitmq:latest
-                ports:
-                    - 5672:5672
-
         steps:
+            - name: debug
+              run: |
+                  echo ${{ secrets.DOCKER_USERNAME }} > /tmp/aa.txt
+                  echo $(cat /tmp/aa.txt)
 
-            - name: Check out app
-              uses: actions/checkout@v3
-
-            - name: Set up Python
-              uses: actions/setup-python@v4
-              with:
-                  python-version: ${{ matrix.python-version }}
-                  cache: pip
-                  cache-dependency-path: |
-                      **/setup.cfg
-                      **/pyproject.toml
-                      **/requirements*.txt
-            - name: Install package
-              run: pip install -e .[dev] aiida-core==${{ matrix.aiida-core-version }}
-
-            - name: Run pytest
-              run: pytest -v tests --cov
-              env:
-                  TAG: ${{ matrix.tag }}
-
-            - name: Upload coverage reports to Codecov
-              uses: codecov/codecov-action@v3
+            - name: Login to GitHub Container Registry 🔑
+              uses: docker/login-action@v3
               with:
-                  flags: python-${{ matrix.python-version }}
+                  registry: docker.io
+                  username: ${{ secrets.DOCKER_USERNAME }}
+                  password: ${{ secrets.DOCKER_PASSWORD }}
+
+    build-amd64:
+        uses: ./.github/workflows/docker-build.yml
+        with:
+            runsOn: ubuntu-22.04
+            platforms: linux/amd64
+        secrets: inherit
+
+
+    #test-amd64:
+    #    needs: build-amd64
+    #    strategy:
+    #        fail-fast: false
+    #    uses: ./.github/workflows/docker-test.yml
+    #    with:
+    #        runsOn: ubuntu-22.04
+    #        image: ${{ needs.build-amd64.outputs.image }}
+    #        integration: false

.github/workflows/docker-build.yml

@@ -0,0 +1,85 @@
+---
+name: Build images and upload them to ghcr.io
+
+env:
+    BUILDKIT_PROGRESS: plain
+
+on:
+    workflow_call:
+        inputs:
+            runsOn:
+                description: GitHub Actions Runner image
+                required: true
+                type: string
+            platforms:
+                description: Target platforms for the build (linux/amd64 and/or linux/arm64)
+                required: true
+                type: string
+        secrets:
+            REGISTRY_USERNAME:
+                required: true
+            REGISTRY_TOKEN:
+                required: true
+        outputs:
+            image:
+                description: Images identified by digests
+                value: ${{ jobs.build.outputs.image }}
+
+jobs:
+    build:
+        name: ${{ inputs.platforms }}
+        runs-on: ${{ inputs.runsOn }}
+        timeout-minutes: 120
+
+        outputs:
+            image: ${{ steps.bake_metadata.outputs.image }}
+
+        # Make sure we fail if any command in a piped command sequence fails
+        defaults:
+            run:
+                shell: bash -e -o pipefail {0}
+
+        steps:
+
+            - name: Checkout Repo ⚡️
+              uses: actions/checkout@v4
+
+            - name: Set up QEMU
+              if: ${{ inputs.platforms != 'linux/amd64' }}
+              uses: docker/setup-qemu-action@v3
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+
+            - name: Login to GitHub Container Registry 🔑
+              uses: docker/login-action@v3
+              with:
+                  registry: ghcr.io
+                  username: ${{ secrets.REGISTRY_USERNAME }}
+                  password: ${{ secrets.REGISTRY_TOKEN }}
+
+            - name: Build and upload to ghcr.io 📤
+              id: build-upload
+              uses: docker/bake-action@v4
+              with:
+                  workdir: docker
+                  push: true
+                  # Using provenance to disable default attestation so it will build only desired images:
+                  # https://github.com/orgs/community/discussions/45969
+                  provenance: false
+                  set: |
+                      *.platform=${{ inputs.platforms }}
+                      *.output=type=registry,push-by-digest=true,name-canonical=true
+                      *.cache-to=type=gha,scope=${{ github.workflow }},mode=max
+                      *.cache-from=type=gha,scope=${{ github.workflow }}
+                  files: |
+                      docker-bake.hcl
+                      build.json
+                      ../.github/workflows/env.hcl
+
+            - name: Set output variables
+              id: bake_metadata
+              run: |
+                  .github/workflows/extract-image-name.sh | tee -a "${GITHUB_OUTPUT}"
+              env:
+                  BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }}

.github/workflows/docker-publish.yml

@@ -0,0 +1,92 @@
+---
+name: Publish images to Docker container registries
+
+env:
+    # https://github.com/docker/metadata-action?tab=readme-ov-file#environment-variables
+    DOCKER_METADATA_PR_HEAD_SHA: true
+
+on:
+    workflow_call:
+        inputs:
+            runsOn:
+                description: GitHub Actions Runner image
+                required: true
+                type: string
+            images:
+                description: Images built in build step
+                required: true
+                type: string
+            registry:
+                description: Docker container registry
+                required: true
+                type: string
+
+jobs:
+
+    release:
+        runs-on: ${{ inputs.runsOn }}
+        timeout-minutes: 30
+        strategy:
+            fail-fast: true
+            matrix:
+                target: [base, base-with-services, lab, full-stack]
+
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Login to GitHub Container Registry 🔑
+              uses: docker/login-action@v3
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Login to DockerHub 🔑
+              uses: docker/login-action@v3
+              if: inputs.registry == 'docker.io'
+              with:
+                  registry: docker.io
+                  username: ${{ secrets.DOCKER_USERNAME }}
+                  password: ${{ secrets.DOCKER_PASSWORD }}
+
+            - name: Read build variables
+              id: build_vars
+              run: |
+                  vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries')
+                  echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}"
+
+            - name: Docker meta
+              id: meta
+              uses: docker/metadata-action@v5
+              env: ${{ fromJSON(steps.build_vars.outputs.vars) }}
+              with:
+                  # e.g. ghcr.io/aiidalab/full-stack
+                  images: ${{ inputs.registry }}/${{ github.repository_owner }}/${{ matrix.target }}
+                  tags: |
+                      type=ref,event=pr
+                      type=edge,enable={{is_default_branch}}
+                      type=raw,value=aiida-${{ env.AIIDA_VERSION }},enable=${{ github.ref_type == 'tag'  && startsWith(github.ref_name, 'v') }}
+                      type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }}
+                      type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }}
+                      type=match,pattern=v(\d{4}\.\d{4}(-.+)?),group=1
+
+            - name: Determine source image
+              id: images
+              run: |
+                  src=$(echo '${{ inputs.images }}'| jq -cr '.[("${{ matrix.target }}"|ascii_upcase|sub("-"; "_"; "g")) + "_IMAGE"]')
+                  echo "src=$src" | tee -a "${GITHUB_OUTPUT}"
+
+            - name: Push image
+              uses: akhilerm/[email protected]
+              with:
+                  src: ${{ steps.images.outputs.src }}
+                  dst: ${{ steps.meta.outputs.tags }}
+
+            - name: Docker Hub Description
+              if: inputs.registry == 'docker.io'
+              uses: peter-evans/dockerhub-description@v4
+              with:
+                  username: ${{ secrets.DOCKER_USERNAME }}
+                  password: ${{ secrets.DOCKER_PASSWORD }}
+                  repository: aiidalab/${{ matrix.target }}
+                  short-description: ${{ github.event.repository.description }}

.github/workflows/docker-test.yml

@@ -0,0 +1,57 @@
+---
+name: Test newly built images
+
+on:
+    workflow_call:
+        inputs:
+            runsOn:
+                description: GitHub Actions Runner image
+                required: true
+                type: string
+            images:
+                description: Images built in build step
+                required: true
+                type: string
+            target:
+                description: Target image for testing
+                required: false
+                type: string
+            integration:
+                description: Run integration tests
+                required: false
+                type: boolean
+
+jobs:
+
+    test:
+        name: ${{ inputs.integration && inputs.runsOn || inputs.target }}
+        runs-on: ${{ inputs.runsOn }}
+        timeout-minutes: 20
+
+        steps:
+
+            - name: Checkout Repo ⚡️
+              uses: actions/checkout@v4
+
+            - name: Login to GitHub Container Registry 🔑
+              uses: docker/login-action@v3
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Set Up Python 🐍
+              if: ${{ startsWith(inputs.runsOn, 'ubuntu') }}
+              uses: actions/setup-python@v5
+              with:
+                  python-version: '3.11'
+                  cache: pip
+
+            - name: Install dependencies 📦
+              run: |
+                  pip install -r requirements.txt
+                  pip freeze
+
+            - name: Run tests
+              run: pytest -m "${{ inputs.integration && 'integration' || 'not integration' }}" --target ${{inputs.target}}
+              env: ${{ fromJSON(inputs.images) }}

.github/workflows/env.hcl

@@ -0,0 +1,2 @@
+# env.hcl
+REGISTRY = "ghcr.io"

.github/workflows/extract-image-name.sh

@@ -0,0 +1,34 @@
+# Extract image names together with their sha256 digests
+# from the docker/bake-action metadata output.
+# These together uniquely identify newly built images.
+
+# The input to this script is a JSON string passed via BAKE_METADATA env variable
+# Here's example input (trimmed to relevant bits):
+# BAKE_METADATA: {
+#    "base": {
+#      "containerimage.descriptor": {
+#        "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+#        "digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
+#        "size": 6170,
+#      },
+#      "containerimage.digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
+#      "image.name": "ghcr.io/aiidalab/qe"
+#    }
+#  }
+#
+# Example output (real output is on one line):
+#
+# image="ghcr.io/aiidalab/qe@sha256:79a0f984b9e03b733304fda809ad3e8eec8416992ff334052d75da00cadb8f12"
+# }
+#
+# This json output is later turned to environment variables using fromJson() GHA builtin
+# (e.g. BUILD_MACHINE_IMAGE=ghcr.io/aiidalab/qe@sha256:8e57a52b...)
+# and these are in turn read in the docker-compose.<target>.yml files for tests.
+
+if [[ -z ${BAKE_METADATA-} ]];then
+    echo "ERROR: Environment variable BAKE_METADATA is not set!"
+    exit 1
+fi
+
+image=$(echo "${BAKE_METADATA}" | jq -c '. as $base | to_entries[] | [(.value."image.name"|split(",")[0]),(.value."containerimage.digest")]|join("@")' | tr -d '"')
+echo "image=$image"