Skip to content

Commit

Permalink
Merge pull request #44 from CExA-project/feature/improve-ci
Browse files Browse the repository at this point in the history
Improve CI
  • Loading branch information
yasahi-hpc authored Feb 9, 2024
2 parents dd796b1 + 05e6626 commit 03f8a3a
Show file tree
Hide file tree
Showing 8 changed files with 122 additions and 63 deletions.
45 changes: 25 additions & 20 deletions .github/workflows/__build_base.yaml
Original file line number Diff line number Diff line change
@@ -1,19 +1,24 @@
# Generate base images for each backend. Images are stored on Github registry
# as Docker format inconditionnaly and as Singularity format only if requested.
# Docker images are used for building Kokkos FFT, while Singularity images are
# used for test execution on the righteous hardware. This workflow can only be
# invoked through another workflows.
# Generate base images for each compiler environment. Images are stored on
# Github registry as Docker format inconditionnaly and as Singularity format
# only if requested. Docker images are used for building Kokkos FFT, while
# Singularity images are used for test execution on the righteous hardware.
# This workflow can only be invoked through another workflows.

name: Build base images

on:
workflow_call:
inputs:
# suffix of the Docker and Singularity images
image_name_suffix:
image_suffix:
description: "Suffix of the Docker and Singularity images"
required: false
default: main
type: string
image_tag:
description: "Tag of the Docker and Singularity images"
required: false
default: latest
type: string

env:
# Force the use of BuildKit for Docker
Expand All @@ -25,14 +30,14 @@ jobs:

strategy:
matrix:
backend:
- name: openmp
image:
- name: gcc
use_singularity: false
- name: cuda
- name: nvcc
use_singularity: true
- name: hip
- name: rocm
use_singularity: false
- name: sycl
- name: intel
use_singularity: false

steps:
Expand Down Expand Up @@ -61,19 +66,19 @@ jobs:
- name: Build Docker image
run: |
docker build \
-t ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_${{ inputs.image_name_suffix }} \
--cache-from ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_main \
-t ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.image.name }}_${{ inputs.image_suffix }}:${{ inputs.image_tag }} \
--cache-from ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.image.name }}_main:latest \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--progress=plain \
docker/${{ matrix.backend.name }}
docker/${{ matrix.image.name }}
- name: Push Docker image
run: docker push ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_${{ inputs.image_name_suffix }}
run: docker push ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.image.name }}_${{ inputs.image_suffix }}:${{ inputs.image_tag }}

- name: Convert Docker image to Singularity
run: singularity build base.sif docker://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_${{ inputs.image_name_suffix }}
if: ${{ matrix.backend.use_singularity }}
run: singularity build base.sif docker://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.image.name }}_${{ inputs.image_suffix }}:${{ inputs.image_tag }}
if: ${{ matrix.image.use_singularity }}

- name: Push Singularity image
run: singularity push base.sif oras://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_singularity_${{ inputs.image_name_suffix }}
if: ${{ matrix.backend.use_singularity }}
run: singularity push base.sif oras://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.image.name }}_singularity_${{ inputs.image_suffix }}:${{ inputs.image_tag }}
if: ${{ matrix.image.use_singularity }}
57 changes: 57 additions & 0 deletions .github/workflows/__check_docker_files.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Check if Docker images have changed. In that case, proposes images suffix and
# tag.

name: Check Docker files

on:
workflow_call:
inputs:
event_name:
description: "Event name of the calling workflow"
required: true
type: string

outputs:
docker_files_have_changed:
description: "True if any Docker file was modified"
value: ${{ jobs.check_docker_files.outputs.docker_files_have_changed }}
image_suffix:
description: "Suffix of the images"
value: ${{ jobs.check_docker_files.outputs.image_suffix }}
image_tag:
description: "Tag of the images"
value: ${{ jobs.check_docker_files.outputs.image_tag }}

jobs:
check_docker_files:
runs-on: ubuntu-latest

outputs:
# true if any Docker file was modified in the PR (PR mode) or since last pushed commit (push mode)
docker_files_have_changed: ${{ steps.get_changed_docker_files.outputs.any_changed == 'true' }}
# use "pr" as image name suffix if on PR mode and if any Docker file was modified, otherwise use "main";
# this is intended to avoid a PR to alter Docker images for other PRs or for the main branch
image_suffix: ${{ steps.get_changed_docker_files.outputs.any_changed == 'true' && inputs.event_name == 'pull_request' && 'pr' || 'main' }}
# use "<hash>" as image name tag if on PR mode and if any Docker file was modified, otherwise use "latest";
# this is intended to distinguish PR images from each other
image_tag: ${{ steps.get_changed_docker_files.outputs.any_changed == 'true' && inputs.event_name == 'pull_request' && github.sha || 'latest' }}

steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Get changed Docker files
id: get_changed_docker_files
uses: tj-actions/changed-files@v42
with:
files: docker/**/Dockerfile

- name: List changed Docker files
if: ${{ steps.get_changed_docker_files.outputs.any_changed == 'true' }}
env:
ALL_CHANGED_FILES: ${{ steps.get_changed_docker_files.outputs.all_changed_files }}
run: |
for file in $ALL_CHANGED_FILES; do
echo "$file was changed"
done
59 changes: 21 additions & 38 deletions .github/workflows/build_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,34 +29,10 @@ jobs:
clangFormatVersion: 12

check_docker_files:
runs-on: ubuntu-latest

outputs:
# true if any Docker file was modified in the PR (PR mode) or since last pushed commit (push mode)
docker_files_have_changed: ${{ steps.get_changed_docker_files.outputs.any_changed == 'true' }}
# use "pr" as image name suffix if on PR mode and if any Docker file was modified, otherwise use "main"
# this is intended to avoid a PR test to alter Docker images for other PRs or for the main branch
image_name_suffix: ${{ steps.get_changed_docker_files.outputs.any_changed == 'true' && github.event_name == 'pull_request' && 'pr' || 'main' }}

steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
uses: ./.github/workflows/__check_docker_files.yaml

- name: Get changed Dockerfiles
id: get_changed_docker_files
uses: tj-actions/changed-files@v42
with:
files: docker/**/Dockerfile

- name: List changed Dockerfiles
if: ${{ steps.docker_files_have_changed.outputs.any_changed == 'true' }}
env:
ALL_CHANGED_FILES: ${{ steps.docker_files_have_changed.outputs.all_changed_files }}
run: |
for file in "$ALL_CHANGED_FILES"; do
echo "$file was changed"
done
with:
event_name: ${{ github.event_name }}

build_base:
needs: check_docker_files
Expand All @@ -66,7 +42,8 @@ jobs:
uses: ./.github/workflows/__build_base.yaml

with:
image_name_suffix: ${{ needs.check_docker_files.outputs.image_name_suffix }}
image_suffix: ${{ needs.check_docker_files.outputs.image_suffix }}
image_tag: ${{ needs.check_docker_files.outputs.image_tag }}

build:
runs-on: ubuntu-latest
Expand All @@ -82,18 +59,22 @@ jobs:
matrix:
backend:
- name: openmp
image: gcc
c_compiler: gcc
cxx_compiler: g++
cmake_flags: -DKokkos_ENABLE_OPENMP=ON
- name: cuda
image: nvcc
c_compiler: gcc
cxx_compiler: g++
cmake_flags: -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON
- name: hip
image: rocm
c_compiler: hipcc
cxx_compiler: hipcc
cmake_flags: -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_VEGA90A=ON
- name: sycl
image: intel
c_compiler: icx
cxx_compiler: icpx
# building for Intel PVC was unsuccessful without the proper device
Expand Down Expand Up @@ -124,7 +105,7 @@ jobs:

- name: Configure
run: |
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_${{ needs.check_docker_files.outputs.image_name_suffix }} \
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake -B build \
-DCMAKE_INSTALL_PREFIX=/work/install \
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
Expand All @@ -139,7 +120,7 @@ jobs:
- name: Build
run: |
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_${{ needs.check_docker_files.outputs.image_name_suffix }} \
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake --build build -j $(( $(nproc) * 2 + 1 ))
- name: Prepare artifacts
Expand All @@ -157,21 +138,21 @@ jobs:

- name: Install
run: |
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_${{ needs.check_docker_files.outputs.image_name_suffix }} \
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake --install build
- name: Configure and build test code
# Use the built and installed Kokkos FFT library to build a test code
run: |
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_${{ needs.check_docker_files.outputs.image_name_suffix }} \
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake -B build_test \
-DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \
-DCMAKE_C_COMPILER=${{ matrix.backend.c_compiler }} \
-DCMAKE_CXX_COMPILER=${{ matrix.backend.cxx_compiler }} \
-DCMAKE_CXX_STANDARD=17 \
-DCMAKE_PREFIX_PATH=/work/install \
install_test
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_${{ needs.check_docker_files.outputs.image_name_suffix }} \
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake --build build_test -j $(( $(nproc) * 2 + 1 ))
test:
Expand All @@ -189,9 +170,11 @@ jobs:
backend:
# run CUDA tests on Ruche supercomputer
- name: cuda
image: nvcc
runner: [self-hosted, cuda]
# run OpenMP tests on Azure server
- name: openmp
image: gcc
runner: ubuntu-latest

steps:
Expand All @@ -206,18 +189,18 @@ jobs:

- name: Pull Singularity image
# pulling the image in advance seems necessary as sometimes invoking `singularity run` on the image URL fails because it cannot find ghcr.io
run: singularity pull oras://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_singularity_${{ needs.check_docker_files.outputs.image_name_suffix }}:latest
run: singularity pull oras://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_singularity_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }}
if: ${{ matrix.backend.name == 'cuda' }}

- name: Run CUDA tests within Slurm job and Singularity image
run: |
srun --nodes=1 --time=01:00:00 -p gpua100 --gres=gpu:1 \
singularity run --nv --bind $PWD/build:/work/build -H /work/build base_${{ matrix.backend.name }}_singularity_${{ needs.check_docker_files.outputs.image_name_suffix }}_latest.sif \
ctest
singularity run --nv --bind $PWD/build:/work/build -H /work/build base_${{ matrix.backend.image }}_singularity_${{ needs.check_docker_files.outputs.image_suffix }}_${{ needs.check_docker_files.outputs.image_tag }}.sif \
ctest --output-on-failure
if: ${{ matrix.backend.name == 'cuda' }}

- name: Run OpenMP tests within Docker image
run: |
docker run -v $PWD/build:/work/build -w /work/build ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_${{ needs.check_docker_files.outputs.image_name_suffix }} \
ctest
docker run -v $PWD/build:/work/build -w /work/build ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
ctest --output-on-failure
if: ${{ matrix.backend.name == 'openmp' }}
22 changes: 18 additions & 4 deletions .github/workflows/pre_build_base.yaml
Original file line number Diff line number Diff line change
@@ -1,14 +1,28 @@
# Recreate the base images inconditionnaly on a regural basis. This is to
# ensure that anybody using the Docker files for local development does not
# encounter weird bugs unnoticed by the CI, because the images would be too
# old.
# Recreate the base images inconditionnaly on a regural basis and on push on
# the main branch. This is to ensure that anybody using the Docker files for
# local development does not encounter weird bugs unnoticed by the CI, because
# the images would be too old.

name: Pre-build base images

on:
schedule:
- cron: "0 1 2,16 * *" # every 2nd and 16th of the month at 1am UTC
push:
branches:
- main

jobs:
check_docker_files:
uses: ./.github/workflows/__check_docker_files.yaml

with:
event_name: ${{ github.event_name }}

build_base:
needs: check_docker_files

# run inconditionnaly on schedule mode or if Docker files changed on other modes
if: ${{ github.event_name == 'schedule' || needs.check_docker_files.outputs.docker_files_have_changed == 'true' }}

uses: ./.github/workflows/__build_base.yaml
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion docker/hip/Dockerfile → docker/rocm/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
ARG BASE=rocm/dev-ubuntu-20.04:5.4
ARG BASE=rocm/dev-ubuntu-20.04
FROM $BASE

ARG ADDITIONAL_PACKAGES
Expand Down

0 comments on commit 03f8a3a

Please sign in to comment.