Improve tests using Singularity #392

Workflow file for this run

.github/workflows/build_test.yaml at e2d0a18

	# SPDX-FileCopyrightText: (C) The kokkos-fft development team, see COPYRIGHT.md file
	#
	# SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception

	# Build and test Kokkos FFT using Docker and Singularity images. Pre-generated
	# images are pulled from Github registry; they are updated only if the current
	# PR or commit modified the Docker files.

	name: CI

	on:
	pull_request:
	branches:
	- main

	env:
	# Force the use of BuildKit for Docker
	DOCKER_BUILDKIT: 1

	jobs:
	# run linter on the code
	lint:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: DoozyX/[email protected]
	with:
	source: 'common/ fft/ examples/ install_test/'
	exclude: ''
	extensions: 'hpp,cpp'
	clangFormatVersion: 17

	# check if Docker images should be recreated
	check_docker_files:
	uses: ./.github/workflows/__check_docker_files.yaml

	with:
	event_name: ${{ github.event_name }}

	# build Docker images if necessary
	build_base:
	needs: check_docker_files

	if: ${{ needs.check_docker_files.outputs.docker_files_have_changed == 'true' }}

	uses: ./.github/workflows/__build_base.yaml

	with:
	image_suffix: ${{ needs.check_docker_files.outputs.image_suffix }}
	image_tag: ${{ needs.check_docker_files.outputs.image_tag }}

	# build project
	build:
	runs-on: ubuntu-latest

	needs:
	- check_docker_files
	- build_base

	# run this job even if build_base did not run
	# FIXME revert this!
	# if: ${{ ! cancelled() && (needs.build_base.result == 'success' \|\| needs.build_base.result == 'skipped') }}
	if: false

	strategy:
	matrix:
	backend:
	- name: clang-tidy
	image: clang
	compiler:
	c: clang
	cxx: clang++
	cmake_flags:
	cxx_standard: 17
	kokkos: -DKokkos_ENABLE_SERIAL=ON
	kokkos_fft: -DCMAKE_CXX_FLAGS="-Wall -Wextra" -DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_CXX_CLANG_TIDY="clang-tidy;-warnings-as-errors=*"
	benchmark: OFF
	cmake_build_type: Debug
	- name: openmp
	image: gcc
	compiler:
	c: gcc
	cxx: g++
	cmake_flags:
	cxx_standard: 17
	kokkos: -DKokkos_ENABLE_OPENMP=ON -DKokkos_ENABLE_SERIAL=ON
	kokkos_fft: -DCMAKE_CXX_FLAGS="-Wall -Wextra" -DCMAKE_COMPILE_WARNING_AS_ERROR=ON
	benchmark: ON
	cmake_build_type: Debug
	- name: threads
	image: gcc
	compiler:
	c: gcc
	cxx: g++
	cmake_flags:
	cxx_standard: 20
	kokkos: -DKokkos_ENABLE_THREADS=ON
	kokkos_fft: -DCMAKE_CXX_FLAGS="-Wall -Wextra" -DCMAKE_COMPILE_WARNING_AS_ERROR=ON
	benchmark: ON
	cmake_build_type: Release
	- name: serial
	image: gcc
	compiler:
	c: gcc
	cxx: g++
	cmake_flags:
	cxx_standard: 17
	kokkos: -DKokkos_ENABLE_SERIAL=ON
	kokkos_fft: -DCMAKE_CXX_FLAGS="-Wall -Wextra" -DCMAKE_COMPILE_WARNING_AS_ERROR=ON
	benchmark: ON
	cmake_build_type: Release
	- name: cuda
	image: nvcc
	compiler:
	c: gcc
	cxx: g++
	cmake_flags:
	cxx_standard: 17
	kokkos: -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON -DKokkos_ENABLE_OPENMP=ON -DKokkos_ENABLE_SERIAL=ON
	kokkos_fft: -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror"
	benchmark: ON
	cmake_build_type: Release
	- name: hip
	image: rocm
	compiler:
	c: hipcc
	cxx: hipcc
	cmake_flags:
	cxx_standard: 17
	kokkos: -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_VEGA90A=ON -DKokkos_ENABLE_THREADS=ON
	kokkos_fft: -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror"
	benchmark: ON
	cmake_build_type: Release
	- name: rocm
	image: rocm
	compiler:
	c: hipcc
	cxx: hipcc
	cmake_flags:
	cxx_standard: 20
	kokkos: -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_VEGA90A=ON
	kokkos_fft: -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror" -DKokkosFFT_ENABLE_ROCFFT=ON
	benchmark: ON
	cmake_build_type: Release
	- name: sycl
	image: intel
	compiler:
	c: icx
	cxx: icpx
	cmake_flags:
	# building for Intel PVC was unsuccessful without the proper
	# device, so for now, we simply generate generic Intel GPU code
	cxx_standard: 17
	kokkos: -DKokkos_ENABLE_SYCL=ON -DKokkos_ARCH_INTEL_GEN=ON
	kokkos_fft: -DCMAKE_CXX_FLAGS="-Wall -Wextra"
	benchmark: ON
	cmake_build_type: Release
	target:
	- name: native
	cmake_flags: ""
	- name: host_device
	cmake_flags: -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON
	exclude:
	- backend:
	name: clang-tidy
	target:
	name: host_device
	- backend:
	name: openmp
	target:
	name: host_device
	- backend:
	name: threads
	target:
	name: host_device
	- backend:
	name: serial
	target:
	name: host_device

	steps:
	- name: Free Disk Space (Ubuntu)
	uses: jlumbroso/[email protected]
	with:
	tool-cache: true
	large-packages: false

	- name: Checkout built branch
	uses: actions/checkout@v4
	with:
	submodules: recursive

	- name: Configure
	run: \|
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake -B build \
	-DCMAKE_BUILD_TYPE=${{ matrix.backend.cmake_build_type }} \
	-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
	-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
	-DCMAKE_CXX_STANDARD=${{ matrix.backend.cmake_flags.cxx_standard }} \
	-DKokkosFFT_ENABLE_EXAMPLES=ON \
	-DKokkosFFT_ENABLE_BENCHMARK=${{ matrix.backend.cmake_flags.benchmark }} \
	-DKokkosFFT_ENABLE_TESTS=ON \
	-DKokkosFFT_ENABLE_INTERNAL_KOKKOS=ON \
	${{ matrix.backend.cmake_flags.kokkos }} \
	${{ matrix.backend.cmake_flags.kokkos_fft }} \
	${{ matrix.target.cmake_flags }}

	- name: Build
	run: \|
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake --build build -j $(( $(nproc) * 2 + 1 ))

	- name: Prepare artifacts
	# this is mandatory to preserve execution rights
	run: tar -cvf tests_${{ matrix.backend.name }}.tar build/
	if: ${{ matrix.target.name == 'native' }}

	- name: Save artifacts
	uses: actions/upload-artifact@v4
	with:
	name: tests_${{ matrix.backend.name }}
	path: tests_${{ matrix.backend.name }}.tar
	if: ${{ matrix.target.name == 'native' }}

	# consume the project in a dummy project
	# ideally, this part should be executed in a different job using almost
	# the same `matrix` (minus `target`) as the `build` job and having the
	# same `needs` and `if` as the `test` job; this would however make a
	# duplicated and error-prone `matrix`, and sharing a `matrix` between
	# jobs is currently not possible with GitHub Actions (neither directly,
	# nor using YAML anchors, other options were too cumbersome when writing
	# this); consequently, this part is just here for now

	- name: Prepare as subdirectory
	run: \|
	# prepare kokkos
	git clone ./tpls/kokkos ./install_test/as_subdirectory/tpls/kokkos
	# prepare kokkos-fft
	git clone . ./install_test/as_subdirectory/tpls/kokkos-fft

	- name: Configure and build test project as subdirectory
	run: \|
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake -B install_test/as_subdirectory/build \
	-DCMAKE_BUILD_TYPE=${{ matrix.backend.cmake_build_type }} \
	-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
	-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
	-DCMAKE_CXX_STANDARD=${{ matrix.backend.cmake_flags.cxx_standard }} \
	${{ matrix.backend.cmake_flags.kokkos }} \
	${{ matrix.backend.cmake_flags.kokkos_fft }} \
	${{ matrix.target.cmake_flags }} \
	install_test/as_subdirectory
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake --build install_test/as_subdirectory/build

	- name: Configure, build and install Kokkos as library
	run: \|
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake -B build_kokkos \
	-DCMAKE_INSTALL_PREFIX=/work/install \
	-DCMAKE_BUILD_TYPE=${{ matrix.backend.cmake_build_type }} \
	-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
	-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
	-DCMAKE_CXX_STANDARD=${{ matrix.backend.cmake_flags.cxx_standard }} \
	${{ matrix.backend.cmake_flags.kokkos }} \
	tpls/kokkos
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake --build build_kokkos -j $(( $(nproc) * 2 + 1 ))
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake --install build_kokkos

	- name: Configure, build and install KokkosFFT as library
	run: \|
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake -B build_kokkos_fft \
	-DCMAKE_INSTALL_PREFIX=/work/install\
	-DCMAKE_PREFIX_PATH=/work/install \
	-DCMAKE_BUILD_TYPE=${{ matrix.backend.cmake_build_type }} \
	-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
	-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
	-DCMAKE_CXX_STANDARD=${{ matrix.backend.cmake_flags.cxx_standard }} \
	${{ matrix.backend.cmake_flags.kokkos_fft }} \
	${{ matrix.target.cmake_flags }}
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake --build build_kokkos_fft -j $(( $(nproc) * 2 + 1 ))
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake --install build_kokkos_fft

	- name: Configure and build test project as library
	run: \|
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake -B install_test/as_library/build \
	-DCMAKE_BUILD_TYPE=${{ matrix.backend.cmake_build_type }} \
	-DCMAKE_PREFIX_PATH=/work/install \
	-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
	-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
	-DCMAKE_CXX_STANDARD=${{ matrix.backend.cmake_flags.cxx_standard }} \
	install_test/as_library
	docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	cmake --build install_test/as_library/build -j $(( $(nproc) * 2 + 1 ))

	# test the project
	test:
	runs-on: ${{ matrix.backend.runner }}

	needs:
	- check_docker_files
	- build

	# run this job even if build_base did not run
	# FIXME revert this!
	# if: ${{ ! cancelled() && needs.build.result == 'success' }}
	if: true

	strategy:
	matrix:
	backend:
	# run CUDA tests on Ruche supercomputer
	- name: cuda
	image: nvcc
	runner: [self-hosted, cuda]
	# FIXME revert this!
	# # run OpenMP tests on Azure server
	# - name: openmp
	# image: gcc
	# runner: ubuntu-latest
	# # run Threads tests on Azure server
	# - name: threads
	# image: gcc
	# runner: ubuntu-latest
	# # run Serial tests on Azure server
	# - name: serial
	# image: gcc
	# runner: ubuntu-latest

	steps:
	# FIXME revert this!
	# - name: Get artifacts
	# uses: actions/download-artifact@v4
	# with:
	# name: tests_${{ matrix.backend.name }}
	#
	# - name: Deploy artifacts
	# run: tar -xvf tests_${{ matrix.backend.name }}.tar

	- name: Login in GitHub Containers Repository with Singularity
	run: \|
	run \
	-m "singularity/3.8.3/gcc-11.2.0" \
	bash -c "echo ${{ secrets.GITHUB_TOKEN }} \| singularity remote login -u ${{ github.actor }} --password-stdin oras://ghcr.io"
	if: ${{ matrix.backend.name == 'cuda' }}

	- name: Pull Singularity image
	# pulling the image in advance seems necessary as sometimes invoking `singularity run` on the image URL fails because it cannot find ghcr.io
	run: \|
	run \
	-m "singularity/3.8.3/gcc-11.2.0" \
	singularity pull oras://ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_singularity_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }}
	if: ${{ matrix.backend.name == 'cuda' }}

	- name: Run CUDA tests within Slurm job and Singularity image
	run: \|
	run \
	-g a100 \
	-m "singularity/3.8.3/gcc-11.2.0" \
	singularity run --nv --bind $PWD/build:/work/build -H /work/build base_${{ matrix.backend.image }}_singularity_${{ needs.check_docker_files.outputs.image_suffix }}_${{ needs.check_docker_files.outputs.image_tag }}.sif \
	ctest --output-on-failure
	if: ${{ matrix.backend.name == 'cuda' }}

	- name: Run OpenMP/Threads/Serial tests within Docker image
	run: \|
	docker run -v $PWD/build:/work/build -w /work/build ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
	ctest --output-on-failure
	if: ${{ matrix.backend.name == 'openmp' \|\| matrix.backend.name == 'threads' \|\| matrix.backend.name == 'serial' }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Improve tests using Singularity #392

Workflow file

Improve tests using Singularity #392

Jobs

Run details

Workflow file for this run