diff --git a/.github/actions/create-dev-env/action.yml b/.github/actions/create-dev-env/action.yml deleted file mode 100644 index 85c46980..00000000 --- a/.github/actions/create-dev-env/action.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: Build environment -description: Create build environment - -inputs: - architecture: - description: architecture to be run on - required: true - type: string - -runs: - using: composite - steps: - # actions/setup-python doesn't support Linux arm64 runners - # See: https://github.com/actions/setup-python/issues/108 - # python3 is manually preinstalled in the arm64 VM self-hosted runner - - name: Set Up Python 🐍 - if: ${{ inputs.architecture == 'amd64' }} - uses: actions/setup-python@v4 - with: - python-version: 3.11 - - - name: Install Dev Dependencies 📦 - run: | - pip install --upgrade pip - pip install --upgrade -r requirements-dev.txt - shell: bash diff --git a/.github/actions/load-image/action.yml b/.github/actions/load-image/action.yml deleted file mode 100644 index 4a8665f9..00000000 --- a/.github/actions/load-image/action.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -name: Load Docker image -description: Download image tar and load it to docker - -inputs: - image: - description: Image name - required: true - type: string - architecture: - description: Image architecture - required: true - type: string - -runs: - using: composite - steps: - - name: Download built image 📥 - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.image }}-${{ inputs.architecture }} - path: /tmp/aiidalab - - name: Load downloaded image to docker 📥 - run: | - docker load --input /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}.tar - docker image ls --all - shell: bash - - name: Delete the file 🗑️ - run: rm -f /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}.tar - shell: bash - if: always() diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 23a311ee..ce0d77c8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ updates: - package-ecosystem: github-actions directory: / schedule: - interval: daily + interval: monthly groups: gha-dependencies: patterns: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..8e65a758 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,77 @@ +--- +name: Build images and upload them to ghcr.io + +env: + BUILDKIT_PROGRESS: plain + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + platforms: + description: Target platforms for the build (linux/amd64 and/or linux/arm64) + required: true + type: string + outputs: + images: + description: Images identified by digests + value: ${{ jobs.build.outputs.images }} + +jobs: + build: + name: ${{ inputs.platforms }} + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 120 + + outputs: + images: ${{ steps.bake_metadata.outputs.images }} + + # Make sure we fail if any command in a piped command sequence fails + defaults: + run: + shell: bash -e -o pipefail {0} + + steps: + + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Set up QEMU + if: ${{ inputs.platforms != 'linux/amd64' }} + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and upload to ghcr.io 📤 + id: build-upload + uses: docker/bake-action@v4 + with: + push: true + # Using provenance to disable default attestation so it will build only desired images: + # https://github.com/orgs/community/discussions/45969 + provenance: false + set: | + *.platform=${{ inputs.platforms }} + *.output=type=registry,push-by-digest=true,name-canonical=true + files: | + docker-bake.hcl + build.json + .github/workflows/env.hcl + + - name: Set output variables + id: bake_metadata + run: | + .github/workflows/extract-image-names.sh | tee -a "${GITHUB_OUTPUT}" + env: + BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }} diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml deleted file mode 100644 index 0c59fe08..00000000 --- a/.github/workflows/docker-build-test-upload.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- -name: Build a new image and test it; then upload the image, tags and manifests to GitHub artifacts - -env: - OWNER: ${{ github.repository_owner }} - -on: - workflow_call: - inputs: - image: - description: Image name - required: true - type: string - architecture: - description: Image architecture, e.g. amd64, arm64 - required: true - type: string - runsOn: - description: GitHub Actions Runner image - required: true - type: string - -jobs: - build-test-upload: - runs-on: ${{ inputs.runsOn }} - timeout-minutes: 20 - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - name: Create dev environment 📦 - uses: ./.github/actions/create-dev-env - with: - architecture: ${{ inputs.architecture }} - - # Self-hosted runners share a state (whole VM) between runs - # Also, they might have running or stopped containers, - # which are not cleaned up by `docker system prun` - - name: Reset docker state and cleanup artifacts 🗑️ - if: ${{ inputs.platform != 'x86_64' }} - run: | - docker kill $(docker ps --quiet) || true - docker rm $(docker ps --all --quiet) || true - docker system prune --all --force - rm -rf /tmp/aiidalab/ - shell: bash - - - name: Build image 🛠 - run: doit build --target ${{ inputs.image }} --arch ${{ inputs.architecture }} --organization ${{ env.OWNER }} - env: - # Full logs for CI build - BUILDKIT_PROGRESS: plain - shell: bash - - - name: Run tests ✅ - run: VERSION=newly-build pytest -s --target ${{ inputs.image }} - shell: bash - - - name: Save image as a tar for later use 💾 - run: | - mkdir -p /tmp/aiidalab/ - docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}.tar - shell: bash - if: always() - - - name: Upload image as artifact 💾 - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.image }}-${{ inputs.architecture }} - path: /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}.tar - retention-days: 3 - if-no-files-found: error - if: always() diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml deleted file mode 100644 index b0d43fec..00000000 --- a/.github/workflows/docker-merge-tags.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -name: Download images tags from GitHub artifacts and create multi-platform manifests - -on: - workflow_call: - inputs: - image: - description: Image name - required: true - type: string - registry: - description: Docker registry, e.g. ghcr.io, docker.io - required: true - type: string - secrets: - REGISTRY_USERNAME: - required: true - REGISTRY_TOKEN: - required: true - - -jobs: - merge-tags: - runs-on: ubuntu-latest - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - name: Create dev environment 📦 - uses: ./.github/actions/create-dev-env - with: - architecture: amd64 - - - name: Download amd64 tags file 📥 - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.registry }}-${{ inputs.image }}-amd64-tags - path: /tmp/aiidalab - - name: Download arm64 tags file 📥 - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.registry }}-${{ inputs.image }}-arm64-tags - path: /tmp/aiidalab - - - name: Login to Container Registry 🔑 - uses: docker/login-action@v2 - with: - registry: ${{ inputs.registry }} - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Merge tags for the images of different arch 🔀 - run: | - for arch_tag in $(cat /tmp/aiidalab/${{ inputs.image }}-amd64-tags.txt); do - tag=$(echo $arch_tag | sed "s/:amd64-/:/") - docker manifest create $tag --amend $arch_tag - docker manifest push $tag - done - - for arch_tag in $(cat /tmp/aiidalab/${{ inputs.image }}-arm64-tags.txt); do - tag=$(echo $arch_tag | sed "s/:arm64-/:/") - docker manifest create $tag --amend $arch_tag - docker manifest push $tag - done - shell: bash diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml deleted file mode 100644 index c264b1b1..00000000 --- a/.github/workflows/docker-push.yml +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: Download Docker image and its tags from GitHub artifacts, apply them and push the image to container registry - -env: - OWNER: ${{ github.repository_owner }} - -on: - workflow_call: - inputs: - image: - description: Image name - required: true - type: string - architecture: - description: Image architecture - required: true - type: string - registry: - description: Docker registry - required: true - type: string - secrets: - REGISTRY_USERNAME: - required: true - REGISTRY_TOKEN: - required: true - -jobs: - tag-push: - runs-on: ubuntu-latest - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - name: Create dev environment 📦 - uses: ./.github/actions/create-dev-env - with: - architecture: ${{ inputs.architecture }} - - name: Load image to Docker 📥 - uses: ./.github/actions/load-image - with: - image: ${{ inputs.image }} - architecture: ${{ inputs.architecture }} - - - name: Read build variables - id: build_vars - run: | - vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') - echo "vars=$vars" >> "${GITHUB_OUTPUT}" - - - name: Docker meta 📝 - id: meta - uses: docker/metadata-action@v4 - env: ${{ fromJson(steps.build_vars.outputs.vars) }} - with: - images: | - name=${{ inputs.registry }}/${{ env.OWNER }}/${{ inputs.image }} - tags: | - type=edge,enable={{is_default_branch}} - type=sha,enable=${{ github.ref_type != 'tag' }} - type=ref,event=pr - type=match,pattern=v(\d{4}\.\d{4}),group=1 - type=raw,value={{tag}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=aiida-${{ env.AIIDA_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - - - name: Login to Container Registry 🔑 - uses: docker/login-action@v2 - with: - registry: ${{ inputs.registry }} - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Set tags for image and push 🏷️📤💾 - run: | - declare -a arr=(${{ steps.meta.outputs.tags }}) - for tag in "${arr[@]}"; do - arch_tag=$(echo ${tag} | sed "s/:/:${{ inputs.architecture }}-/") - docker tag aiidalab/${{ inputs.image }}:newly-build ${arch_tag} - docker push ${arch_tag} - - # write tag to file - mkdir -p /tmp/aiidalab/ - echo ${arch_tag} >> /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}-tags.txt - done - shell: bash - - - name: Upload tags file 📤 - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.registry }}-${{ inputs.image }}-${{ inputs.architecture }}-tags - path: /tmp/aiidalab/${{ inputs.image }}-${{ inputs.architecture }}-tags.txt - retention-days: 3 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index d9d4480d..00000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,180 +0,0 @@ ---- -name: Build, test and push Docker Images - -on: - pull_request: - push: - branches: - - main - tags: - - "v*" - workflow_dispatch: - -# https://docs.github.com/en/actions/using-jobs/using-concurrency -concurrency: - # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - amd64-base: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: base - architecture: amd64 - runsOn: ubuntu-latest - - amd64-base-with-services: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: base-with-services - architecture: amd64 - runsOn: ubuntu-latest - needs: [amd64-base] - - amd64-lab: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: lab - architecture: amd64 - runsOn: ubuntu-latest - needs: [amd64-base] - - amd64-full-stack: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: full-stack - architecture: amd64 - runsOn: ubuntu-latest - needs: [amd64-base-with-services, amd64-lab] - - arm64-base: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: base - architecture: arm64 - runsOn: ARM64 - - arm64-lab: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: lab - architecture: arm64 - runsOn: ARM64 - needs: [arm64-base] - - arm64-base-with-services: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: base-with-services - architecture: arm64 - runsOn: ARM64 - needs: [arm64-base] - - arm64-full-stack: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: full-stack - architecture: arm64 - runsOn: ARM64 - needs: [arm64-base-with-services, arm64-lab] - - amd64-push-ghcr: - if: always() - uses: ./.github/workflows/docker-push.yml - strategy: - matrix: - image: ["base", "base-with-services", "lab", "full-stack"] - with: - architecture: amd64 - image: ${{ matrix.image }} - registry: ghcr.io - secrets: - REGISTRY_USERNAME: ${{ github.actor }} - REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack] - - arm64-push-ghcr: - if: always() - uses: ./.github/workflows/docker-push.yml - strategy: - matrix: - image: ["base", "base-with-services", "lab", "full-stack"] - with: - architecture: arm64 - image: ${{ matrix.image }} - registry: ghcr.io - secrets: - REGISTRY_USERNAME: ${{ github.actor }} - REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack] - - merge-tags-ghcr: - if: always() - uses: ./.github/workflows/docker-merge-tags.yml - strategy: - matrix: - image: ["base", "base-with-services", "lab", "full-stack"] - with: - image: ${{ matrix.image }} - registry: ghcr.io - secrets: - REGISTRY_USERNAME: ${{ github.actor }} - REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: [amd64-push-ghcr, arm64-push-ghcr] - - amd64-push-dockerhub: - if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) - uses: ./.github/workflows/docker-push.yml - strategy: - matrix: - image: ["base", "base-with-services", "lab", "full-stack"] - with: - architecture: amd64 - image: ${{ matrix.image }} - registry: docker.io - secrets: - REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }} - needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack] - - arm64-push-dockerhub: - if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) - uses: ./.github/workflows/docker-push.yml - strategy: - matrix: - image: ["base", "base-with-services", "lab", "full-stack"] - with: - architecture: arm64 - image: ${{ matrix.image }} - registry: docker.io - secrets: - REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }} - needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack] - - merge-tags-dockerhub: - if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) - uses: ./.github/workflows/docker-merge-tags.yml - strategy: - matrix: - image: ["base", "base-with-services", "lab", "full-stack"] - with: - image: ${{ matrix.image }} - registry: docker.io - secrets: - REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }} - needs: [amd64-push-dockerhub, arm64-push-dockerhub] - - release: - runs-on: ubuntu-latest - needs: [merge-tags-ghcr, merge-tags-dockerhub] - steps: - - uses: actions/checkout@v4 - - - name: Create release - uses: softprops/action-gh-release@v1 - with: - generate_release_notes: true - if: github.repository == 'aiidalab/aiidalab-docker-stack' && startsWith(github.ref, 'refs/tags/v') diff --git a/.github/workflows/env.hcl b/.github/workflows/env.hcl new file mode 100644 index 00000000..fc2b844e --- /dev/null +++ b/.github/workflows/env.hcl @@ -0,0 +1,2 @@ +# env.hcl +REGISTRY = "ghcr.io/" diff --git a/.github/workflows/extract-image-names.sh b/.github/workflows/extract-image-names.sh new file mode 100755 index 00000000..9aca2de8 --- /dev/null +++ b/.github/workflows/extract-image-names.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -euo pipefail + +# Extract image names together with their sha256 digests +# from the docker/bake-action metadata output. +# These together uniquely identify newly built images. + +# The input to this script is a JSON string passed via BAKE_METADATA env variable +# Here's example input (trimmed to relevant bits): +# BAKE_METADATA: { +# "base": { +# "containerimage.descriptor": { +# "mediaType": "application/vnd.docker.distribution.manifest.v2+json", +# "digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", +# "size": 6170, +# }, +# "containerimage.digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", +# "image.name": "ghcr.io/aiidalab/base" +# }, +# "base-with-services": { +# "image.name": "ghcr.io/aiidalab/base-with-services" +# "containerimage.digest": "sha256:6753a809b5b2675bf4c22408e07c1df155907a465b33c369ef93ebcb1c4fec26", +# "...": "" +# } +# "full-stack": { +# "image.name": "ghcr.io/aiidalab/full-stack" +# "containerimage.digest": "sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48", +# "...": "" +# } +# "lab": { +# "image.name": "ghcr.io/aiidalab/lab" +# "containerimage.digest": "sha256:4d9be090da287fcdf2d4658bb82f78bad791ccd15dac9af594fb8306abe47e97", +# "...": "" +# } +# } +# +# Example output (real output is on one line): +# +# images={ +# "BASE_IMAGE": "ghcr.io/aiidalab/base@sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", +# "BASE_WITH_SERVICES_IMAGE": "ghcr.io/aiidalab/base-with-services@sha256:6753a809b5b2675bf4c22408e07c1df155907a465b33c369ef93ebcb1c4fec26", +# "FULL_STACK_IMAGE": "ghcr.io/aiidalab/full-stack@sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48", +# "LAB_IMAGE": "ghcr.io/aiidalab/lab@sha256:4d9be090da287fcdf2d4658bb82f78bad791ccd15dac9af594fb8306abe47e97" +# } +# +# This json output is later turned to environment variables using fromJson() GHA builtin +# (e.g. BASE_IMAGE=ghcr.io/aiidalab/base@sha256:8e57a52b...) +# and these are in turn read in the docker-compose..yml files for tests. + +if [[ -z ${BAKE_METADATA-} ]];then + echo "ERROR: Environment variable BAKE_METADATA is not set!" + exit 1 +fi + +images=$(echo "${BAKE_METADATA}" | jq -c '. as $base |[to_entries[] |{"key": (.key|ascii_upcase|sub("-"; "_"; "g") + "_IMAGE"), "value": [(.value."image.name"|split(",")[0]),.value."containerimage.digest"]|join("@")}] |from_entries') +echo "images=$images" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..a7e7195b --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,92 @@ +--- +name: Docker + +on: + pull_request: + push: + branches: + - main + tags: + - "v*" + workflow_dispatch: + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + +jobs: + + build-amd64: + uses: ./.github/workflows/build.yml + with: + runsOn: ubuntu-22.04 + platforms: linux/amd64 + + test-amd64: + needs: build-amd64 + strategy: + fail-fast: false + matrix: + target: ["base", "lab", "base-with-services", "full-stack"] + uses: ./.github/workflows/test.yml + with: + runsOn: ubuntu-22.04 + images: ${{ needs.build-amd64.outputs.images }} + target: ${{ matrix.target }} + integration: false + + build: + needs: test-amd64 + uses: ./.github/workflows/build.yml + with: + runsOn: ubuntu-22.04 + platforms: linux/amd64,linux/arm64 + + # To save self-hosted runner resources, we're only testing full-stack image + test-arm64: + needs: build + uses: ./.github/workflows/test.yml + with: + runsOn: buildjet-4vcpu-ubuntu-2204-arm + images: ${{ needs.build.outputs.images }} + target: full-stack + integration: false + + test-integration: + needs: build + strategy: + fail-fast: false + matrix: + runner: [ubuntu-22.04, buildjet-4vcpu-ubuntu-2204-arm] + uses: ./.github/workflows/test.yml + with: + runsOn: ${{ matrix.runner }} + images: ${{ needs.build.outputs.images }} + target: full-stack + integration: true + + publish-ghcr: + needs: [build, test-amd64, test-arm64] + uses: ./.github/workflows/publish.yml + with: + runsOn: ubuntu-22.04 + images: ${{ needs.build.outputs.images }} + registry: ghcr.io + secrets: inherit + + publish-dockerhub: + if: >- + github.repository == 'aiidalab/aiidalab-docker-stack' + && (github.ref_type == 'tag' || github.ref_name == 'main') + needs: [build, publish-ghcr] + uses: ./.github/workflows/publish.yml + with: + runsOn: ubuntu-22.04 + images: ${{ needs.build.outputs.images }} + registry: docker.io + secrets: inherit diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..49ddb15e --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,92 @@ +--- +name: Publish images to Docker container registries + +env: + # https://github.com/docker/metadata-action?tab=readme-ov-file#environment-variables + DOCKER_METADATA_PR_HEAD_SHA: true + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + images: + description: Images built in build step + required: true + type: string + registry: + description: Docker container registry + required: true + type: string + +jobs: + + release: + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 30 + strategy: + fail-fast: true + matrix: + target: ["base", "base-with-services", "lab", "full-stack"] + + steps: + - uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub 🔑 + uses: docker/login-action@v3 + if: inputs.registry == 'docker.io' + with: + registry: docker.io + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Read build variables + id: build_vars + run: | + vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') + echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}" + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + env: ${{ fromJSON(steps.build_vars.outputs.vars) }} + with: + # e.g. ghcr.io/aiidalab/full-stack + images: ${{ inputs.registry }}/${{ github.repository_owner }}/${{ matrix.target }} + tags: | + type=ref,event=pr + type=edge,enable={{is_default_branch}} + type=raw,value=aiida-${{ env.AIIDA_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=match,pattern=v(\d{4}\.\d{4}(-.+)?),group=1 + + - name: Determine source image + id: images + run: | + src=$(echo '${{ inputs.images }}'| jq -cr '.[("${{ matrix.target }}"|ascii_upcase|sub("-"; "_"; "g")) + "_IMAGE"]') + echo "src=$src" | tee -a "${GITHUB_OUTPUT}" + + - name: Push image + uses: akhilerm/tag-push-action@v2.2.0 + with: + src: ${{ steps.images.outputs.src }} + dst: ${{ steps.meta.outputs.tags }} + + - name: Docker Hub Description + if: inputs.registry == 'docker.io' + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: aiidalab/${{ matrix.target }} + short-description: ${{ github.event.repository.description }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..9fb79317 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,57 @@ +--- +name: Test newly built images + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + images: + description: Images built in build step + required: true + type: string + target: + description: Target image for testing + required: false + type: string + integration: + description: Run integration tests + required: false + type: boolean + +jobs: + + test: + name: ${{ inputs.integration && inputs.runsOn || inputs.target }} + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 20 + + steps: + + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Up Python 🐍 + if: ${{ startsWith(inputs.runsOn, 'ubuntu') }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + + - name: Install dependencies 📦 + run: | + pip install -r requirements.txt + pip freeze + + - name: Run tests + run: pytest -m "${{ inputs.integration && 'integration' || 'not integration' }}" --target ${{inputs.target}} + env: ${{ fromJSON(inputs.images) }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f665917b..3836638f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,6 @@ repos: rev: v4.5.0 hooks: - id: check-json - - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace @@ -24,3 +23,9 @@ repos: - id: ruff-format - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] + + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: "0.27.3" + hooks: + - id: check-dependabot + - id: check-github-workflows diff --git a/aarch64-runner/setup.sh b/aarch64-runner/setup.sh index 78fb4387..e7b3eb2c 100755 --- a/aarch64-runner/setup.sh +++ b/aarch64-runner/setup.sh @@ -71,4 +71,5 @@ brew install docker brew install docker-compose brew install docker-buildx brew install colima +brew install jq EOF diff --git a/docker-bake.hcl b/docker-bake.hcl index 81fa26a7..46d8225b 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -26,7 +26,6 @@ variable "ORGANIZATION" { } variable "REGISTRY" { - default = "docker.io/" } variable "PLATFORMS" { @@ -40,10 +39,17 @@ variable "TARGETS" { function "tags" { params = [image] result = [ - "${REGISTRY}${ORGANIZATION}/${image}:newly-build", + "${REGISTRY}${ORGANIZATION}/${image}${VERSION}", ] } +# Get a Python version string without the patch version (e.g. "3.9.13" -> "3.9") +# Used to construct paths to Python site-packages folder. +function "get_python_minor_version" { + params = [python_version] + result = join(".", slice(split(".", "${python_version}"), 0, 2)) +} + group "default" { targets = "${TARGETS}" } @@ -83,9 +89,6 @@ target "base-with-services" { "PGSQL_VERSION" = "${PGSQL_VERSION}" } } -# PYTHON_MINOR_VERSION is a Python version string -# without the patch version (e.g. "3.9") -# Used to construct paths to Python site-packages folder. target "lab" { inherits = ["lab-meta"] context = "stack/lab" @@ -96,7 +99,7 @@ target "lab" { args = { "AIIDALAB_VERSION" = "${AIIDALAB_VERSION}" "AIIDALAB_HOME_VERSION" = "${AIIDALAB_HOME_VERSION}" - "PYTHON_MINOR_VERSION" = join(".", slice(split(".", "${PYTHON_VERSION}"), 0, 2)) + "PYTHON_MINOR_VERSION" = get_python_minor_version("${PYTHON_VERSION}") } } target "full-stack" { diff --git a/dodo.py b/dodo.py index 99d53d42..d5d493f8 100644 --- a/dodo.py +++ b/dodo.py @@ -46,7 +46,7 @@ _VERSION_PARAM = { "name": "version", "long": "version", - "type": "str", + "type": str, "default": VERSION, "help": ( "Specify the version of the stack for building / testing. Defaults to a " @@ -62,6 +62,15 @@ "help": "Specify the platform to build for. Examples: arm64, amd64.", } +_TARGET_PARAM = { + "name": "targets", + "long": "targets", + "short": "t", + "type": list, + "default": [], + "help": "Specify the target to build.", +} + def task_build(): """Build all docker images.""" @@ -70,18 +79,18 @@ def generate_version_override( version, registry, targets, architecture, organization ): platforms = [f"linux/{architecture}"] - - Path("docker-bake.override.json").write_text( - json.dumps( - { - "VERSION": version, - "REGISTRY": registry, - "TARGETS": targets, - "ORGANIZATION": organization, - "PLATFORMS": platforms, - } - ) - ) + overrides = { + "VERSION": f":{version}", + "REGISTRY": registry, + "ORGANIZATION": organization, + "PLATFORMS": platforms, + } + # If no targets are specifies via cmdline, we'll build all images, + # as specified in docker-bake.hcl + if targets: + overrides["TARGETS"] = targets + + Path("docker-bake.override.json").write_text(json.dumps(overrides)) return { "actions": [ @@ -96,14 +105,7 @@ def generate_version_override( _REGISTRY_PARAM, _VERSION_PARAM, _ARCH_PARAM, - { - "name": "targets", - "long": "targets", - "short": "t", - "type": list, - "default": [], - "help": "Specify the target to build.", - }, + _TARGET_PARAM, ], "verbosity": 2, } @@ -112,9 +114,11 @@ def generate_version_override( def task_tests(): """Run tests with pytest.""" + # TODO: This currently does not work! + # https://github.com/aiidalab/aiidalab-docker-stack/issues/451 return { "actions": ["REGISTRY=%(registry)s VERSION=:%(version)s pytest -v"], - "params": [_REGISTRY_PARAM, _VERSION_PARAM], + "params": [_REGISTRY_PARAM, _VERSION_PARAM, _TARGET_PARAM], "verbosity": 2, } diff --git a/requirements-dev.txt b/requirements.txt similarity index 100% rename from requirements-dev.txt rename to requirements.txt diff --git a/stack/base-with-services/Dockerfile b/stack/base-with-services/Dockerfile index 1a987e63..99f0331e 100644 --- a/stack/base-with-services/Dockerfile +++ b/stack/base-with-services/Dockerfile @@ -10,35 +10,34 @@ ARG AIIDA_VERSION ARG PGSQL_VERSION ARG TARGETARCH -RUN mamba create -p /opt/conda/envs/aiida-core-services --yes \ - postgresql=${PGSQL_VERSION} \ - && mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -# Install RabbitMQ in a dedicated conda environment. -# If the architecture is arm64, we install the default version of rabbitmq provided by the generic binary, -# # https://www.rabbitmq.com/install-generic-unix.html the version needs to be compatible with system's erlang version. +# Install RabbitMQ and PostgreSQL in a dedicated conda environment. +# +# RabbitMQ is not available on conda-forge at the time being, see: +# https://github.com/conda-forge/rabbitmq-server-feedstock/issues/67If +# Instead we need install erlang via apt and RabbitMQ as a "Generic Unix Build", see: +# https://www.rabbitmq.com/install-generic-unix.html +# Note that this version must be compatible with system's erlang version. +# Note that system erlang from arm64 is already installed in the base image, +# together with other APT dependencies to save build time. RUN if [ "$TARGETARCH" = "amd64" ]; then \ - mamba install -p /opt/conda/envs/aiida-core-services --yes \ - rabbitmq-server=3.8.14 && \ + mamba create -p /opt/conda/envs/aiida-core-services --yes \ + postgresql=${PGSQL_VERSION} \ + rabbitmq-server=3.8.14 && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}"; \ elif [ "$TARGETARCH" = "arm64" ]; then \ - apt-get update && apt-get install -y --no-install-recommends \ - erlang && \ - rm -rf /var/lib/apt/lists/* && \ - apt-get clean all && \ + mamba create -p /opt/conda/envs/aiida-core-services --yes \ + postgresql=${PGSQL_VERSION} && \ + mamba clean --all -f -y && \ export RMQ_VERSION=3.9.13 && \ wget -c https://github.com/rabbitmq/rabbitmq-server/releases/download/v${RMQ_VERSION}/rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ tar -xf rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ rm rabbitmq-server-generic-unix-${RMQ_VERSION}.tar.xz && \ mv rabbitmq_server-${RMQ_VERSION} /opt/conda/envs/aiida-core-services/ && \ - fix-permissions "/opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}"; \ ln -sf /opt/conda/envs/aiida-core-services/rabbitmq_server-${RMQ_VERSION}/sbin/* /opt/conda/envs/aiida-core-services/bin/; \ -else \ - echo "Unknown architecture: ${TARGETARCH}."; \ fi # Configure AiiDA profile. @@ -46,9 +45,9 @@ COPY config-quick-setup.yaml . COPY before-notebook.d/20_start-postgresql.sh /usr/local/bin/before-notebook.d/ COPY before-notebook.d/30_start-rabbitmq-${TARGETARCH}.sh /usr/local/bin/before-notebook.d/ -# Supress rabbitmq version warning for arm64 since -# it is built using latest version rabbitmq from apt install. -# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf +# Supress rabbitmq version warning from aiida-core. +# This is needed for the arm64 build which uses RabbitMQ version >3.8, for which +# we explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf COPY before-notebook.d/41_suppress-rabbitmq-version-warning.sh /usr/local/bin/before-notebook.d/ RUN if [ "$TARGETARCH" = "amd64" ]; then \ rm /usr/local/bin/before-notebook.d/41_suppress-rabbitmq-version-warning.sh; \ diff --git a/stack/base/Dockerfile b/stack/base/Dockerfile index b74691aa..6489157f 100644 --- a/stack/base/Dockerfile +++ b/stack/base/Dockerfile @@ -5,17 +5,27 @@ LABEL maintainer="AiiDAlab Team " USER root -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - # for apps which need to install pymatgen: - # https://pymatgen.org/installation.html#installation-tips-for-optional-libraries - build-essential && \ +# build-essential: includes GCC compilers that are needed when building +# pip packages from sources, which often seems to happen for pymatgen: +# https://pymatgen.org/installation.html#installation-tips-for-optional-libraries +# rsync: needed to support the new AiiDA backup command +# povray: rendering engine used in aiidalab-widgets-base +ENV EXTRA_APT_PACKAGES "curl povray rsync build-essential" + +# For ARM64 we need to install erlang as it is not available on conda-forge +# (this is needed later as rabbitmq dependency in base-with-services image, +# but we install it here so that we don't have to invoke apt multiple times. +ARG TARGETARCH +RUN if [ "$TARGETARCH" = "arm64" ]; then \ + EXTRA_APT_PACKAGES="erlang ${EXTRA_APT_PACKAGES}"; \ + fi;\ + apt-get update --yes && \ + apt-get install --yes --no-install-recommends ${EXTRA_APT_PACKAGES} && \ apt-get clean && rm -rf /var/lib/apt/lists/* WORKDIR /opt/ ARG AIIDA_VERSION - # Pin shared requirements in the base environment. # We pin aiida-core to the exact installed version, # to prevent accidental upgrade or downgrade, that might diff --git a/stack/docker-compose.base-with-services.yml b/stack/docker-compose.base-with-services.yml index 9074a8e7..4250d756 100644 --- a/stack/docker-compose.base-with-services.yml +++ b/stack/docker-compose.base-with-services.yml @@ -4,7 +4,7 @@ version: '3.4' services: aiidalab: - image: ${REGISTRY:-}${BASE_WITH_SERVICES_IMAGE:-aiidalab/base-with-services}:${VERSION:-newly-build} + image: ${REGISTRY:-}${BASE_WITH_SERVICES_IMAGE:-aiidalab/base-with-services}${VERSION:-} environment: TZ: Europe/Zurich DOCKER_STACKS_JUPYTER_CMD: notebook diff --git a/stack/docker-compose.base.yml b/stack/docker-compose.base.yml index 3888e526..b24dbac2 100644 --- a/stack/docker-compose.base.yml +++ b/stack/docker-compose.base.yml @@ -25,7 +25,7 @@ services: - aiida-rmq-data:/var/lib/rabbitmq/ aiidalab: - image: ${REGISTRY:-}${BASE_IMAGE:-aiidalab/base}:${VERSION:-newly-build} + image: ${REGISTRY:-}${BASE_IMAGE:-aiidalab/base}${VERSION:-} environment: RMQHOST: messaging TZ: Europe/Zurich diff --git a/stack/docker-compose.full-stack.yml b/stack/docker-compose.full-stack.yml index b21f7538..788c52b6 100644 --- a/stack/docker-compose.full-stack.yml +++ b/stack/docker-compose.full-stack.yml @@ -4,7 +4,7 @@ version: '3.4' services: aiidalab: - image: ${REGISTRY:-}${FULL_STACK_IMAGE:-aiidalab/full-stack}:${VERSION:-newly-build} + image: ${REGISTRY:-}${FULL_STACK_IMAGE:-aiidalab/full-stack}${VERSION:-} environment: TZ: Europe/Zurich DOCKER_STACKS_JUPYTER_CMD: notebook diff --git a/stack/docker-compose.lab.yml b/stack/docker-compose.lab.yml index 5cc6f765..e906d67e 100644 --- a/stack/docker-compose.lab.yml +++ b/stack/docker-compose.lab.yml @@ -25,7 +25,7 @@ services: - aiida-rmq-data:/var/lib/rabbitmq/ aiidalab: - image: ${REGISTRY:-}${LAB_IMAGE:-aiidalab/lab}:${VERSION:-newly-build} + image: ${REGISTRY:-}${LAB_IMAGE:-aiidalab/lab}${VERSION:-} environment: RMQHOST: messaging TZ: Europe/Zurich diff --git a/stack/full-stack/Dockerfile b/stack/full-stack/Dockerfile index 2a39b0ec..aefe2a82 100644 --- a/stack/full-stack/Dockerfile +++ b/stack/full-stack/Dockerfile @@ -9,18 +9,6 @@ COPY --from=base /opt/config-quick-setup.yaml /opt/ COPY --from=base "${CONDA_DIR}/envs/aiida-core-services" "${CONDA_DIR}/envs/aiida-core-services" COPY --from=base /usr/local/bin/before-notebook.d /usr/local/bin/before-notebook.d -# This is needed because we use multi-stage build. -# the erlang package is not available after the first stage. -# After we move base-with-services to a aiida-core repo, we can remove this. -# Note that it is very important to having the TARGETARCH argument here, otherwise the variable is empty. -ARG TARGETARCH -RUN if [ "$TARGETARCH" = "arm64" ]; then \ - # Install erlang. - apt-get update --yes && \ - apt-get install --yes --no-install-recommends erlang && \ - apt-get clean && rm -rf /var/lib/apt/lists/*; \ -fi - RUN fix-permissions "${CONDA_DIR}" RUN fix-permissions "/home/${NB_USER}/.aiida" diff --git a/stack/lab/Dockerfile b/stack/lab/Dockerfile index 565c74e8..15f122c2 100644 --- a/stack/lab/Dockerfile +++ b/stack/lab/Dockerfile @@ -13,14 +13,6 @@ ENV DOCKER_STACKS_JUPYTER_CMD=notebook USER root WORKDIR /opt/ -# Install additional system packages -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - curl \ - povray \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - # Install aiidalab package ARG AIIDALAB_VERSION RUN mamba install --yes \ diff --git a/tests/conftest.py b/tests/conftest.py index 6e9d9854..d66cca61 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,7 +64,7 @@ def docker_compose(docker_services): @pytest.fixture -def aiidalab_exec(docker_compose): +def aiidalab_exec(notebook_service, docker_compose): def execute(command, user=None, **kwargs): if user: command = f"exec -T --user={user} aiidalab {command}" diff --git a/tests/test_full_stack.py b/tests/test_aiidalab_apps.py similarity index 74% rename from tests/test_full_stack.py rename to tests/test_aiidalab_apps.py index 62939272..c30fcd8c 100644 --- a/tests/test_full_stack.py +++ b/tests/test_aiidalab_apps.py @@ -1,6 +1,8 @@ import pytest -# Tests in this file should pass for the following images +pytestmark = pytest.mark.integration +# Integration tests for the full-stack image. +# Here we make sure we can install aiidalab-widgets-base and aiidalab-qe apps TESTED_TARGETS = "full-stack" @@ -14,19 +16,25 @@ def skip_if_incompatible_target(target): @pytest.fixture(scope="function") def generate_aiidalab_install_output(aiidalab_exec, nb_user): + pkg = None + def _generate_aiidalab_install_output(package_name): + nonlocal pkg + pkg = package_name cmd = f"aiidalab install --yes --pre {package_name}" - output = aiidalab_exec(cmd, user=nb_user).strip() + output = aiidalab_exec(cmd, user=nb_user).strip() output += aiidalab_exec("pip check", user=nb_user).strip() - - # Uninstall the package to make sure the test is repeatable - app_name = package_name.split("@")[0] - aiidalab_exec(f"aiidalab uninstall --yes --force {app_name}", user=nb_user) - return output - return _generate_aiidalab_install_output + # Uninstall the package to make sure the test is repeatable. + # NOTE: This will only uninstall the package itself, not its dependencies! + # Since the dependencies are installed via pip, this is basically a pip limitation + # that would be hard to workaround here. + yield _generate_aiidalab_install_output + if pkg: + app_name = pkg.split("@")[0] + aiidalab_exec(f"aiidalab uninstall --yes --force {app_name}", user=nb_user) @pytest.mark.parametrize("package_name", ["aiidalab-widgets-base", "quantum-espresso"])