Misc CI improvements (#1618)

* Fetch git tags when performing documentation builds. Allowing for the version number to appear properly in the generated documentation. This should allow for us to publish the documentation build from CI when performing a release.
* Allow overriding the GIT_URL, useful when performing CI against a commit/branch/tag that exists in a remote other than origin
* Replace list of CUDA architectures with RAPIDS place-holder (we were building for 60 even though we no longer support it)
* Construct the `CMAKE_BUILD_ALL_FEATURES` var in a more readable way
* Allow overriding the build dir, useful for local builds using `USE_HOST_GIT=1` to avoid conflicting with a potentially existing build directory
* Move generated env.yaml to `$WORKSPACE_TMP`, prevents the file from being written to the root of the git repo.
* Rather than init submodules in each stage by hand, use the submodules flag in the checkout action


## By Submitting this PR I confirm:
- I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md).
- When the PR is ready for review, new or existing tests cover these changes.
- When the PR is ready for review, the documentation is up to date with these changes.

Authors:
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #1618

.github/workflows/ci_pipe.yml

@@ -85,6 +85,7 @@ jobs:
           lfs: false
           path: 'morpheus'
           fetch-depth: 0
+          submodules: 'recursive'
 
       - name: Get AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@v1-node16
@@ -115,6 +116,7 @@ jobs:
         with:
           lfs: false
           path: 'morpheus'
+          submodules: 'recursive'
 
       - name: Get AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@v1-node16
@@ -149,6 +151,7 @@ jobs:
         with:
           lfs: false
           path: 'morpheus'
+          submodules: 'recursive'
 
       - name: Get AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@v1-node16
@@ -180,6 +183,9 @@ jobs:
         with:
           lfs: false
           path: 'morpheus'
+          # Fetch tags so that documentation builds for releases will report the version number correctly
+          fetch-tags: true
+          submodules: 'recursive'
 
       - name: Get AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@v1-node16
@@ -213,6 +219,7 @@ jobs:
           lfs: false
           path: 'morpheus'
           fetch-depth: 0
+          submodules: 'recursive'
 
       - name: Get AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@v1-node16

ci/scripts/bootstrap_local_ci.sh

@@ -25,6 +25,8 @@ else
     git checkout ${GIT_BRANCH}
     git pull
     git checkout ${GIT_COMMIT}
+    git fetch --tags
+    git submodule update --init --recursive
 fi
 
 export MORPHEUS_ROOT=$(pwd)

ci/scripts/common.sh

@@ -73,7 +73,7 @@ function get_modified_files() {
    local GIT_DIFF_BASE=${GIT_DIFF_BASE:-$(get_merge_base)}
 
    # If invoked by a git-commit-hook, this will be populated
-   local result=( $(git diff ${GIT_DIFF_ARGS} $(get_merge_base) | grep -P ${1:-'.*'}) )
+   local result=( $(git diff ${GIT_DIFF_ARGS} ${GIT_DIFF_BASE} | grep -P ${1:-'.*'}) )
 
    local files=()
 

ci/scripts/github/build.sh

@@ -21,41 +21,33 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 rapids-dependency-file-generator \
   --output conda \
   --file_key build \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml"
 
-update_conda_env env.yaml
+update_conda_env "${WORKSPACE_TMP}/env.yaml"
 
 log_toolchain
 
-git submodule update --init --recursive
-
 CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_WHEEL=ON"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON"
-if [[ "${LOCAL_CI}" == "" ]]; then
-    CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)"
-fi
 
 rapids-logger "Configuring cmake for Morpheus with ${CMAKE_FLAGS}"
-cmake -B build -G Ninja ${CMAKE_FLAGS} .
+cmake ${CMAKE_FLAGS} .
 
 rapids-logger "Building Morpheus"
-cmake --build build --parallel ${PARALLEL_LEVEL}
+cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL}
 
-if [[ "${LOCAL_CI}" == "" ]]; then
-    rapids-logger "sccache usage for morpheus build:"
-    sccache --show-stats
-fi
+log_sccache_stats
 
 rapids-logger "Archiving results"
-tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" build/dist
+tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" ${BUILD_DIR}/dist
 
-MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/build/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \
+MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \
                 $(find ${MORPHEUS_ROOT}/examples -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
 tar cfj "${WORKSPACE_TMP}/morhpeus_libs.tar.bz" "${MORPHEUS_LIBS[@]}"
 
-CPP_TESTS=($(find ${MORPHEUS_ROOT}/build/morpheus/_lib/tests -name "*.x" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
+CPP_TESTS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/morpheus/_lib/tests -name "*.x" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
 tar cfj "${WORKSPACE_TMP}/cpp_tests.tar.bz" "${CPP_TESTS[@]}"
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}"

ci/scripts/github/checks.sh

@@ -21,42 +21,34 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 rapids-dependency-file-generator \
   --output conda \
   --file_key build \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml"
 
-update_conda_env env.yaml
+update_conda_env "${WORKSPACE_TMP}/env.yaml"
 
 log_toolchain
 
 cd ${MORPHEUS_ROOT}
 
+# Fetching the base branch will try methods that might fail, then fallback to one that does, set +e for this section
+set +e
 fetch_base_branch
-
-git submodule update --init --recursive
+set -e
 
 rapids-logger "Configuring cmake for Morpheus"
 CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF"
 export CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_INPLACE_BUILD=ON"
-if [[ "${LOCAL_CI}" == "" ]]; then
-    CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)"
-fi
 
-cmake -B build -G Ninja ${CMAKE_FLAGS} .
+cmake ${CMAKE_FLAGS} .
 
 rapids-logger "Building Morpheus"
-cmake --build build --parallel ${PARALLEL_LEVEL}
+cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL}
 
-if [[ "${LOCAL_CI}" == "" ]]; then
-    rapids-logger "sccache usage for source build:"
-    sccache --show-stats
-fi
+log_sccache_stats
 
 rapids-logger "Installing Morpheus"
 pip install ./
 
-# Setting this prevents loading of cudf since we don't have a GPU
-export MORPHEUS_IN_SPHINX_BUILD=1
-
 rapids-logger "Checking copyright headers"
 python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT}
 

ci/scripts/github/common.sh

@@ -61,7 +61,26 @@ export SCCACHE_REGION="us-east-2"
 export SCCACHE_IDLE_TIMEOUT=32768
 #export SCCACHE_LOG=debug
 
-export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_PYTHON_BUILD_STUBS=ON -DMORPHEUS_USE_CCACHE=ON"
+# Set the build flags
+export BUILD_DIR=${BUILD_DIR:-build}
+
+_FLAGS=()
+_FLAGS+=("-B" "${BUILD_DIR}")
+_FLAGS+=("-G" "Ninja")
+_FLAGS+=("-DCMAKE_MESSAGE_CONTEXT_SHOW=ON")
+_FLAGS+=("-DMORPHEUS_CUDA_ARCHITECTURES=RAPIDS")
+_FLAGS+=("-DMORPHEUS_USE_CONDA=ON")
+_FLAGS+=("-DMORPHEUS_USE_CCACHE=ON")
+_FLAGS+=("-DMORPHEUS_PYTHON_INPLACE_BUILD=OFF")
+_FLAGS+=("-DMORPHEUS_PYTHON_BUILD_STUBS=ON")
+_FLAGS+=("-DMORPHEUS_BUILD_BENCHMARKS=ON")
+_FLAGS+=("-DMORPHEUS_BUILD_EXAMPLES=ON")
+_FLAGS+=("-DMORPHEUS_BUILD_TESTS=ON")
+if [[ "${LOCAL_CI}" == "" ]]; then
+    _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)")
+fi
+export CMAKE_BUILD_ALL_FEATURES="${_FLAGS[@]}"
+unset _FLAGS
 
 export FETCH_STATUS=0
 
@@ -112,8 +131,11 @@ function fetch_base_branch_gh_api() {
 
 function fetch_base_branch_local() {
     rapids-logger "Retrieving base branch from git"
-    git remote add upstream ${GIT_UPSTREAM_URL}
-    git fetch upstream --tags
+    if [[ "${USE_HOST_GIT}" == "0" ]]; then
+        git remote add upstream ${GIT_UPSTREAM_URL}
+        git fetch upstream --tags
+    fi
+
     source ${MORPHEUS_ROOT}/ci/scripts/common.sh
     export BASE_BRANCH=$(get_base_branch)
     export CHANGE_TARGET="upstream/${BASE_BRANCH}"
@@ -147,6 +169,13 @@ function log_toolchain() {
     sccache --version
 }
 
+function log_sccache_stats() {
+    if [[ "${LOCAL_CI}" == "" ]]; then
+        rapids-logger "sccache usage for morpheus build:"
+        sccache --show-stats
+    fi
+}
+
 function upload_artifact() {
     FILE_NAME=$1
     BASE_NAME=$(basename "${FILE_NAME}")

ci/scripts/github/conda.sh

@@ -23,8 +23,6 @@ cd ${MORPHEUS_ROOT}
 
 fetch_base_branch
 
-git submodule update --init --recursive
-
 # Its important that we are in the base environment for the build
 rapids-logger "Activating Base Conda Environment"
 

ci/scripts/github/docs.sh

@@ -21,33 +21,31 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 rapids-dependency-file-generator \
   --output conda \
   --file_key docs \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml"
 
-update_conda_env env.yaml
+update_conda_env "${WORKSPACE_TMP}/env.yaml"
 
 download_artifact "wheel.tar.bz"
 
 tar xf "${WORKSPACE_TMP}/wheel.tar.bz"
 
-pip install ${MORPHEUS_ROOT}/build/dist/*.whl
+pip install ${MORPHEUS_ROOT}/${BUILD_DIR}/dist/*.whl
 
 rapids-logger "Pulling LFS assets"
 cd ${MORPHEUS_ROOT}
 
 git lfs install
 ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch docs examples
 
-git submodule update --init --recursive
-
 rapids-logger "Configuring for docs"
-cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF -DMORPHEUS_BUILD_DOCS=ON .
+cmake ${CMAKE_BUILD_ALL_FEATURES} -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF -DMORPHEUS_BUILD_DOCS=ON .
 
 rapids-logger "Building docs"
-cmake --build build --parallel ${PARALLEL_LEVEL} --target install
-cmake --build build --parallel ${PARALLEL_LEVEL} --target morpheus_docs
+cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL} --target install
+cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL} --target morpheus_docs
 
 rapids-logger "Archiving the docs"
-tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/docs/html
+tar cfj "${WORKSPACE_TMP}/docs.tar.bz" ${BUILD_DIR}/docs/html
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}"
 set_job_summary_preamble

ci/scripts/github/test.sh

@@ -22,34 +22,26 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 rapids-dependency-file-generator \
   --output conda \
   --file_key test \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml"
 
-update_conda_env env.yaml
+update_conda_env "${WORKSPACE_TMP}/env.yaml"
 
 log_toolchain
 
-git submodule update --init --recursive
-
 CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=ON"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_WHEEL=OFF"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_PERFORM_INSTALL=ON"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX}"
-if [[ "${LOCAL_CI}" == "" ]]; then
-    CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)"
-fi
 
 rapids-logger "Configuring cmake for Morpheus with ${CMAKE_FLAGS}"
-cmake -B build -G Ninja ${CMAKE_FLAGS} .
+cmake ${CMAKE_FLAGS} .
 
 rapids-logger "Building Morpheus"
-cmake --build build --parallel ${PARALLEL_LEVEL} --target install
+cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL} --target install
 
-if [[ "${LOCAL_CI}" == "" ]]; then
-    rapids-logger "sccache usage for morpheus build:"
-    sccache --show-stats
-fi
+log_sccache_stats
 
 rapids-logger "Checking Python stub files"
 
@@ -62,7 +54,7 @@ if [[ $(git status --short --untracked | grep .pyi) != "" ]]; then
     exit 1
 fi
 
-CPP_TESTS=($(find ${MORPHEUS_ROOT}/build -name "*.x"))
+CPP_TESTS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR} -name "*.x"))
 
 rapids-logger "Pulling LFS assets"
 

ci/scripts/run_ci_local.sh

@@ -45,7 +45,10 @@ MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)}
 # match CI, the default)
 USE_HOST_GIT=${USE_HOST_GIT:-0}
 
-GIT_URL=$(git remote get-url origin)
+# Useful when using a host git repo to avoid conflicting with a potentially existing 'build' directory
+BUILD_DIR=${BUILD_DIR:-build-ci}
+
+GIT_URL=${GIT_URL:-$(git remote get-url origin)}
 GIT_URL=$(git_ssh_to_https ${GIT_URL})
 
 GIT_UPSTREAM_URL=$(git remote get-url upstream)
@@ -62,33 +65,41 @@ DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-${CONTAINER_VER}"
 TEST_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-${CONTAINER_VER}"
 
-ENV_LIST="--env LOCAL_CI_TMP=/ci_tmp"
-ENV_LIST="${ENV_LIST} --env GIT_URL=${GIT_URL}"
-ENV_LIST="${ENV_LIST} --env GIT_UPSTREAM_URL=${GIT_UPSTREAM_URL}"
-ENV_LIST="${ENV_LIST} --env GIT_BRANCH=${GIT_BRANCH}"
-ENV_LIST="${ENV_LIST} --env GIT_COMMIT=${GIT_COMMIT}"
-ENV_LIST="${ENV_LIST} --env PARALLEL_LEVEL=$(nproc)"
-ENV_LIST="${ENV_LIST} --env CUDA_VER=${CUDA_VER}"
-ENV_LIST="${ENV_LIST} --env SKIP_CONDA_ENV_UPDATE=${SKIP_CONDA_ENV_UPDATE}"
-ENV_LIST="${ENV_LIST} --env USE_HOST_GIT=${USE_HOST_GIT}"
+ENV_LIST=()
+ENV_LIST+=("--env" "LOCAL_CI_TMP=/ci_tmp")
+ENV_LIST+=("--env" "GIT_URL=${GIT_URL}")
+ENV_LIST+=("--env" "GIT_UPSTREAM_URL=${GIT_UPSTREAM_URL}")
+ENV_LIST+=("--env" "GIT_BRANCH=${GIT_BRANCH}")
+ENV_LIST+=("--env" "GIT_COMMIT=${GIT_COMMIT}")
+ENV_LIST+=("--env" "PARALLEL_LEVEL=$(nproc)")
+ENV_LIST+=("--env" "CUDA_VER=${CUDA_VER}")
+ENV_LIST+=("--env" "SKIP_CONDA_ENV_UPDATE=${SKIP_CONDA_ENV_UPDATE}")
+ENV_LIST+=("--env" "USE_HOST_GIT=${USE_HOST_GIT}")
+ENV_LIST+=("--env" "BUILD_DIR=${BUILD_DIR}")
 
 mkdir -p ${LOCAL_CI_TMP}
 cp ${MORPHEUS_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP}
 
 for STAGE in "${STAGES[@]}"; do
-    DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}"
+    DOCKER_RUN_ARGS=()
+    DOCKER_RUN_ARGS+=("--rm")
+    DOCKER_RUN_ARGS+=("-ti")
+    DOCKER_RUN_ARGS+=("--net=host")
+    DOCKER_RUN_ARGS+=("-v" "${LOCAL_CI_TMP}:/ci_tmp")
+    DOCKER_RUN_ARGS+=("${ENV_LIST[@]}")
+    DOCKER_RUN_ARGS+=("--env STAGE=${STAGE}")
     if [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then
         CONTAINER="${TEST_CONTAINER}"
-        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia"
-        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --gpus all"
-        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --cap-add=sys_nice"
+        DOCKER_RUN_ARGS+=("--runtime=nvidia")
+        DOCKER_RUN_ARGS+=("--gpus all")
+        DOCKER_RUN_ARGS+=("--cap-add=sys_nice")
     else
         CONTAINER="${BUILD_CONTAINER}"
-        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc"
+        DOCKER_RUN_ARGS+=("--runtime=runc")
     fi
 
     if [[ "${USE_HOST_GIT}" == "1" ]]; then
-        DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} -v ${MORPHEUS_ROOT}:/Morpheus"
+        DOCKER_RUN_ARGS+=("-v" "${MORPHEUS_ROOT}:/Morpheus")
     fi
 
     if [[ "${STAGE}" == "bash" ]]; then
@@ -99,7 +110,7 @@ for STAGE in "${STAGES[@]}"; do
 
     echo "Running ${STAGE} stage in ${CONTAINER}"
     set -x
-    docker run ${DOCKER_RUN_ARGS} ${DOCKER_EXTRA_ARGS} ${CONTAINER} ${DOCKER_RUN_CMD}
+    docker run ${DOCKER_RUN_ARGS[@]} ${DOCKER_EXTRA_ARGS} ${CONTAINER} ${DOCKER_RUN_CMD}
     set +x
 
     STATUS=$?