build_test.yaml

# Build and test Kokkos FFT using Docker and Singularity images. Pre-generated
# images are pulled from Github registry; they are updated only if the current
# PR or commit modified the Docker files.

name: CI

on:
  pull_request:
    branches:
      - main

env:
  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
  BUILD_TYPE: Release

  # Force the use of BuildKit for Docker
  DOCKER_BUILDKIT: 1

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: DoozyX/clang-format-lint-action@v0.13
      with:
        source: 'common/ fft/ examples/'
        exclude: ''
        extensions: 'hpp,cpp'
        clangFormatVersion: 12

  check_docker_files:
    uses: ./.github/workflows/__check_docker_files.yaml

    with:
      event_name: ${{ github.event_name }}

  build_base:
    needs: check_docker_files

    if: ${{ needs.check_docker_files.outputs.docker_files_have_changed == 'true' }}

    uses: ./.github/workflows/__build_base.yaml

    with:
      image_suffix: ${{ needs.check_docker_files.outputs.image_suffix }}
      image_tag: ${{ needs.check_docker_files.outputs.image_tag }}

  build:
    runs-on: ubuntu-latest

    needs:
      - check_docker_files
      - build_base

    # run this job even if build_base did not run
    if: ${{ ! cancelled() && (needs.build_base.result == 'success' || needs.build_base.result == 'skipped') }}

    strategy:
      matrix:
        backend:
          - name: openmp
            image: gcc
            c_compiler: gcc
            cxx_compiler: g++
            cmake_flags: -DKokkos_ENABLE_OPENMP=ON
          - name: threads
            image: gcc
            c_compiler: gcc
            cxx_compiler: g++
            cmake_flags: -DKokkos_ENABLE_THREADS=ON
          - name: serial
            image: gcc
            c_compiler: gcc
            cxx_compiler: g++
            cmake_flags: -DKokkos_ENABLE_SERIAL=ON
          - name: cuda
            image: nvcc
            c_compiler: gcc
            cxx_compiler: g++
            cmake_flags: -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON
          - name: hip
            image: rocm
            c_compiler: hipcc
            cxx_compiler: hipcc
            cmake_flags: -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_VEGA90A=ON
          - name: sycl
            image: intel
            c_compiler: icx
            cxx_compiler: icpx
            # building for Intel PVC was unsuccessful without the proper device
            # for now on, we simply use Intel GPU generic code
            cmake_flags: -DKokkos_ENABLE_SYCL=ON -DKokkos_ARCH_INTEL_GEN=ON
        target:
          - name: native
            cmake_flags: ""
          - name: host_device
            cmake_flags: -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON
        exclude:
          - backend:
              name: openmp
            target:
              name: host_device
          - backend:
              name: threads
            target:
              name: host_device
          - backend:
              name: serial
            target:
              name: host_device

    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@v1.2.0
        with:
          tool-cache: true
          large-packages: false

      - name: Checkout built branch
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Configure
        run: |
          docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
            cmake -B build \
            -DCMAKE_INSTALL_PREFIX=/work/install \
            -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
            -DCMAKE_C_COMPILER=${{ matrix.backend.c_compiler }} \
            -DCMAKE_CXX_COMPILER=${{ matrix.backend.cxx_compiler }} \
            -DCMAKE_CXX_STANDARD=17 \
            -DBUILD_TESTING=ON \
            -DKokkosFFT_ENABLE_BENCHMARK=ON \
            -DKokkosFFT_INTERNAL_Kokkos=ON \
            ${{ matrix.backend.cmake_flags }} \
            ${{ matrix.target.cmake_flags }}

      - name: Build
        run: |
          docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
            cmake --build build -j $(( $(nproc) * 2 + 1 ))

      - name: Prepare artifacts
        # this is mandatory to preserve execution rights
        run: tar -cvf tests_${{ matrix.backend.name }}.tar build/
        if: ${{ matrix.target.name == 'native' }}

      - name: Save artifacts
        # use v3 as more recent versions cannot run on Ruche
        uses: actions/upload-artifact@v3
        with:
          name: tests_${{ matrix.backend.name }}
          path: tests_${{ matrix.backend.name }}.tar
        if: ${{ matrix.target.name == 'native' }}

      - name: Install
        run: |
          docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
            cmake --install build

      - name: Configure and build test code
        # Use the built and installed Kokkos FFT library to build a test code
        run: |
          docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
            cmake -B build_test \
            -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \
            -DCMAKE_C_COMPILER=${{ matrix.backend.c_compiler }} \
            -DCMAKE_CXX_COMPILER=${{ matrix.backend.cxx_compiler }} \
            -DCMAKE_CXX_STANDARD=17 \
            -DCMAKE_PREFIX_PATH=/work/install \
            install_test
          docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
            cmake --build build_test -j $(( $(nproc) * 2 + 1 ))

  test:
    runs-on: ${{ matrix.backend.runner }}

    needs:
      - check_docker_files
      - build

    # run this job even if build_base did not run
    if: ${{ ! cancelled() && needs.build.result == 'success' }}

    strategy:
      matrix:
        backend:
          # run CUDA tests on Ruche supercomputer
          - name: cuda
            image: nvcc
            runner: [self-hosted, cuda]
          # run OpenMP tests on Azure server
          - name: openmp
            image: gcc
            runner: ubuntu-latest
          # run Threads tests on Azure server
          - name: threads
            image: gcc
            runner: ubuntu-latest
          # run Serial tests on Azure server
          - name: serial
            image: gcc
            runner: ubuntu-latest

    steps:
      - name: Get artifacts
        # use v3 as more recent versions cannot run on Ruche
        uses: actions/download-artifact@v3
        with:
          name: tests_${{ matrix.backend.name }}

      - name: Deploy artifacts
        run: tar -xvf tests_${{ matrix.backend.name }}.tar

      - name: Pull Singularity image
        # pulling the image in advance seems necessary as sometimes invoking `singularity run` on the image URL fails because it cannot find ghcr.io
        run: singularity pull oras://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_singularity_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }}
        if: ${{ matrix.backend.name == 'cuda' }}

      - name: Run CUDA tests within Slurm job and Singularity image
        run: |
          srun --nodes=1 --time=01:00:00 -p gpua100 --gres=gpu:1 \
          singularity run --nv --bind $PWD/build:/work/build -H /work/build base_${{ matrix.backend.image }}_singularity_${{ needs.check_docker_files.outputs.image_suffix }}_${{ needs.check_docker_files.outputs.image_tag }}.sif \
          ctest --output-on-failure
        if: ${{ matrix.backend.name == 'cuda' }}

      - name: Run OpenMP/Threads/Serial tests within Docker image
        run: |
          docker run -v $PWD/build:/work/build -w /work/build ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
          ctest --output-on-failure
        if: ${{ matrix.backend.name == 'openmp' || matrix.backend.name == 'threads' || matrix.backend.name == 'serial' }}