From 6b06468edeb87eeb3e754cafb6f601a3599e22ed Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Wed, 15 May 2024 23:46:13 +0200 Subject: [PATCH] RWIP: docker workflow refactor GHA --- ...load.yml => _docker-build-test-upload.yml} | 0 ...-merge-tags.yml => _docker-merge-tags.yml} | 0 .../{docker-push.yml => _docker-push.yml} | 0 .github/workflows/ci.yml | 16 ++++ .github/workflows/docker-build.yml | 80 ++++++++++++++++ .github/workflows/docker-publish.yml | 92 +++++++++++++++++++ .github/workflows/docker-test.yml | 57 ++++++++++++ .../workflows/{docker.yml => docker.ymlbak} | 0 .github/workflows/env.hcl | 2 + .github/workflows/extract-image-name.sh | 34 +++++++ docker/Dockerfile | 4 - docker/build.json | 2 +- docker/docker-bake.hcl | 12 +-- 13 files changed, 288 insertions(+), 11 deletions(-) rename .github/workflows/{docker-build-test-upload.yml => _docker-build-test-upload.yml} (100%) rename .github/workflows/{docker-merge-tags.yml => _docker-merge-tags.yml} (100%) rename .github/workflows/{docker-push.yml => _docker-push.yml} (100%) create mode 100644 .github/workflows/docker-build.yml create mode 100644 .github/workflows/docker-publish.yml create mode 100644 .github/workflows/docker-test.yml rename .github/workflows/{docker.yml => docker.ymlbak} (100%) create mode 100644 .github/workflows/env.hcl create mode 100755 .github/workflows/extract-image-name.sh diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/_docker-build-test-upload.yml similarity index 100% rename from .github/workflows/docker-build-test-upload.yml rename to .github/workflows/_docker-build-test-upload.yml diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/_docker-merge-tags.yml similarity index 100% rename from .github/workflows/docker-merge-tags.yml rename to .github/workflows/_docker-merge-tags.yml diff --git a/.github/workflows/docker-push.yml b/.github/workflows/_docker-push.yml similarity index 100% rename from .github/workflows/docker-push.yml rename to .github/workflows/_docker-push.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10ee2f6d1..7eef83d54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,3 +58,19 @@ jobs: uses: codecov/codecov-action@v3 with: flags: python-${{ matrix.python-version }} + + build-amd64: + uses: ./.github/workflows/docker-build.yml + with: + runsOn: ubuntu-22.04 + platforms: linux/amd64 + + #test-amd64: + # needs: build-amd64 + # strategy: + # fail-fast: false + # uses: ./.github/workflows/docker-test.yml + # with: + # runsOn: ubuntu-22.04 + # image: ${{ needs.build-amd64.outputs.image }} + # integration: false diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 000000000..cce0250e0 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,80 @@ +--- +name: Build images and upload them to ghcr.io + +env: + BUILDKIT_PROGRESS: plain + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + platforms: + description: Target platforms for the build (linux/amd64 and/or linux/arm64) + required: true + type: string + outputs: + image: + description: Images identified by digests + value: ${{ jobs.build.outputs.image }} + +jobs: + build: + name: ${{ inputs.platforms }} + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 120 + + outputs: + image: ${{ steps.bake_metadata.outputs.image }} + + # Make sure we fail if any command in a piped command sequence fails + defaults: + run: + shell: bash -e -o pipefail {0} + + steps: + + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Set up QEMU + if: ${{ inputs.platforms != 'linux/amd64' }} + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and upload to ghcr.io 📤 + id: build-upload + uses: docker/bake-action@v4 + with: + workdir: docker + push: true + # Using provenance to disable default attestation so it will build only desired images: + # https://github.com/orgs/community/discussions/45969 + provenance: false + set: | + *.platform=${{ inputs.platforms }} + *.output=type=registry,push-by-digest=true,name-canonical=true + *.cache-to=type=gha,scope=${{ github.workflow }},mode=max + *.cache-from=type=gha,scope=${{ github.workflow }} + files: | + docker-bake.hcl + build.json + ../.github/workflows/env.hcl + + - name: Set output variables + id: bake_metadata + run: | + .github/workflows/extract-image-name.sh | tee -a "${GITHUB_OUTPUT}" + env: + BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..70b0d2b2a --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,92 @@ +--- +name: Publish images to Docker container registries + +env: + # https://github.com/docker/metadata-action?tab=readme-ov-file#environment-variables + DOCKER_METADATA_PR_HEAD_SHA: true + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + images: + description: Images built in build step + required: true + type: string + registry: + description: Docker container registry + required: true + type: string + +jobs: + + release: + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 30 + strategy: + fail-fast: true + matrix: + target: [base, base-with-services, lab, full-stack] + + steps: + - uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub 🔑 + uses: docker/login-action@v3 + if: inputs.registry == 'docker.io' + with: + registry: docker.io + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Read build variables + id: build_vars + run: | + vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') + echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}" + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + env: ${{ fromJSON(steps.build_vars.outputs.vars) }} + with: + # e.g. ghcr.io/aiidalab/full-stack + images: ${{ inputs.registry }}/${{ github.repository_owner }}/${{ matrix.target }} + tags: | + type=ref,event=pr + type=edge,enable={{is_default_branch}} + type=raw,value=aiida-${{ env.AIIDA_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=raw,value=postgresql-${{ env.PGSQL_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + type=match,pattern=v(\d{4}\.\d{4}(-.+)?),group=1 + + - name: Determine source image + id: images + run: | + src=$(echo '${{ inputs.images }}'| jq -cr '.[("${{ matrix.target }}"|ascii_upcase|sub("-"; "_"; "g")) + "_IMAGE"]') + echo "src=$src" | tee -a "${GITHUB_OUTPUT}" + + - name: Push image + uses: akhilerm/tag-push-action@v2.2.0 + with: + src: ${{ steps.images.outputs.src }} + dst: ${{ steps.meta.outputs.tags }} + + - name: Docker Hub Description + if: inputs.registry == 'docker.io' + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: aiidalab/${{ matrix.target }} + short-description: ${{ github.event.repository.description }} diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml new file mode 100644 index 000000000..9fb793176 --- /dev/null +++ b/.github/workflows/docker-test.yml @@ -0,0 +1,57 @@ +--- +name: Test newly built images + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + images: + description: Images built in build step + required: true + type: string + target: + description: Target image for testing + required: false + type: string + integration: + description: Run integration tests + required: false + type: boolean + +jobs: + + test: + name: ${{ inputs.integration && inputs.runsOn || inputs.target }} + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 20 + + steps: + + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Up Python 🐍 + if: ${{ startsWith(inputs.runsOn, 'ubuntu') }} + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + + - name: Install dependencies 📦 + run: | + pip install -r requirements.txt + pip freeze + + - name: Run tests + run: pytest -m "${{ inputs.integration && 'integration' || 'not integration' }}" --target ${{inputs.target}} + env: ${{ fromJSON(inputs.images) }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.ymlbak similarity index 100% rename from .github/workflows/docker.yml rename to .github/workflows/docker.ymlbak diff --git a/.github/workflows/env.hcl b/.github/workflows/env.hcl new file mode 100644 index 000000000..9a8782af0 --- /dev/null +++ b/.github/workflows/env.hcl @@ -0,0 +1,2 @@ +# env.hcl +REGISTRY = "ghcr.io" diff --git a/.github/workflows/extract-image-name.sh b/.github/workflows/extract-image-name.sh new file mode 100755 index 000000000..c470eee4c --- /dev/null +++ b/.github/workflows/extract-image-name.sh @@ -0,0 +1,34 @@ +# Extract image names together with their sha256 digests +# from the docker/bake-action metadata output. +# These together uniquely identify newly built images. + +# The input to this script is a JSON string passed via BAKE_METADATA env variable +# Here's example input (trimmed to relevant bits): +# BAKE_METADATA: { +# "base": { +# "containerimage.descriptor": { +# "mediaType": "application/vnd.docker.distribution.manifest.v2+json", +# "digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", +# "size": 6170, +# }, +# "containerimage.digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", +# "image.name": "ghcr.io/aiidalab/qe" +# } +# } +# +# Example output (real output is on one line): +# +# image="ghcr.io/aiidalab/qe@sha256:79a0f984b9e03b733304fda809ad3e8eec8416992ff334052d75da00cadb8f12" +# } +# +# This json output is later turned to environment variables using fromJson() GHA builtin +# (e.g. BUILD_MACHINE_IMAGE=ghcr.io/aiidalab/qe@sha256:8e57a52b...) +# and these are in turn read in the docker-compose..yml files for tests. + +if [[ -z ${BAKE_METADATA-} ]];then + echo "ERROR: Environment variable BAKE_METADATA is not set!" + exit 1 +fi + +image=$(echo "${BAKE_METADATA}" | jq -c '. as $base | to_entries[] | [(.value."image.name"|split(",")[0]),(.value."containerimage.digest")]|join("@")' | tr -d '"') +echo "image=$image" diff --git a/docker/Dockerfile b/docker/Dockerfile index 1df916f35..e215c95a1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,10 +25,6 @@ RUN cd ${PREINSTALL_APP_FOLDER} && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# The app version is used for installing the app when first time the container is started. -ARG APP_VERSION -ENV APP_VERSION ${APP_VERSION} - ARG QE_VERSION ENV QE_VERSION ${QE_VERSION} RUN mamba create -p /opt/conda/envs/quantum-espresso --yes \ diff --git a/docker/build.json b/docker/build.json index 44dc1b92a..d057e8f7c 100644 --- a/docker/build.json +++ b/docker/build.json @@ -1,6 +1,6 @@ { "variable": { - "AIIDALAB_BASE_IMAGE": { + "BASE_IMAGE": { "default": "ghcr.io/aiidalab/full-stack:2024.1019" }, "QE_VERSION": { diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl index 97017844f..44801e905 100644 --- a/docker/docker-bake.hcl +++ b/docker/docker-bake.hcl @@ -1,21 +1,21 @@ -# docker-bake.hcl for building QeApp images -group "default" { - targets = ["qe"] -} +# docker-bake.hcl variable "QE_VERSION" { } variable "BASE_IMAGE" { - default = "aiidalab/full-stack:latest" } variable "ORGANIZATION" { default = "aiidalab" } +group "default" { + targets = ["qe"] +} + target "qe" { - tags = ["${ORGANIZATION}/qe:newly-baked"] + tags = ["${REGISTRY}/${ORGANIZATION}/qe"] context = "." contexts = { src = ".."