Merge branch 'branch-24.06' of github.com:nv-morpheus/Morpheus into c…

…i-doca

.devcontainer/Dockerfile

@@ -13,6 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rapidsai/devcontainers:23.12-cpp-cuda12.1-mambaforge-ubuntu22.04 AS base
+FROM rapidsai/devcontainers:23.12-cpp-mambaforge-ubuntu22.04 AS base
+
+ENV MORPHEUS_SUPPORT_DOCA=ON
+
+COPY ./docker/optional_deps docker/optional_deps
+RUN ./docker/optional_deps/doca.sh /tmp/doca
 
 ENV PATH="${PATH}:/workspaces/morpheus/.devcontainer/bin"

.devcontainer/devcontainer.json

@@ -18,6 +18,7 @@
     "build": {
         "dockerfile": "Dockerfile"
     },
+    "privileged": true,
 
     "hostRequirements": {
         "gpu": true
@@ -33,7 +34,8 @@
     ],
 
     "runArgs": [
-        "--network=morpheus"
+        "--network=morpheus",
+        "-v=/dev/hugepages:/dev/hugepages"
     ],
 
     "containerEnv": {
@@ -93,8 +95,7 @@
             "type": "bind",
             "source": "${localWorkspaceFolder}/.devcontainer/opt/morpheus",
             "target": "/opt/morpheus"
-        },
-
+        }
     ],
 
     "features": {

.devcontainer/docker/optional_deps/doca.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF}
+LINUX_DISTRO=${LINUX_DISTRO:-ubuntu}
+LINUX_VER=${LINUX_VER:-22.04}
+DOCA_VERSION=${DOCA_VERSION:-2.7.0}
+
+# Exit early if nothing to do
+if [[ ${MORPHEUS_SUPPORT_DOCA} != @(TRUE|ON) ]]; then
+   exit 0
+fi
+
+WORKING_DIR=$1
+
+echo "Installing DOCA using directory: ${WORKING_DIR}"
+
+DEB_DIR=${WORKING_DIR}/deb
+
+mkdir -p ${DEB_DIR}
+
+DOCA_OS_VERSION="ubuntu2204"
+DOCA_PKG_LINK="https://www.mellanox.com/downloads/DOCA/DOCA_v${DOCA_VERSION}/host/doca-host_${DOCA_VERSION}-204000-24.04-${DOCA_OS_VERSION}_amd64.deb"
+
+# Upgrade the base packages (diff between image and Canonical upstream repo)
+apt update -y
+apt upgrade -y
+
+# Install wget
+apt install -y --no-install-recommends wget
+
+wget -qO - ${DOCA_PKG_LINK} -O doca-host.deb
+apt install ./doca-host.deb
+apt update
+apt install -y doca-all
+apt install -y doca-gpu doca-gpu-dev
+
+# Now install the gdrcopy library according to: https://github.com/NVIDIA/gdrcopy
+GDRCOPY_DIR=${WORKING_DIR}/gdrcopy
+
+if [[ ! -d "${GDRCOPY_DIR}" ]] ; then
+    git clone https://github.com/NVIDIA/gdrcopy.git ${GDRCOPY_DIR}
+    cd ${GDRCOPY_DIR}
+else
+    cd ${GDRCOPY_DIR}
+    git pull https://github.com/NVIDIA/gdrcopy.git
+fi
+
+make lib lib_install

.github/workflows/ci_pipe.yml

@@ -85,6 +85,7 @@ jobs:
           lfs: false
           path: 'morpheus'
           fetch-depth: 0
+          submodules: 'recursive'
 
       - name: Get AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@v1-node16
@@ -115,6 +116,7 @@ jobs:
         with:
           lfs: false
           path: 'morpheus'
+          submodules: 'recursive'
 
       - name: Get AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@v1-node16
@@ -149,6 +151,7 @@ jobs:
         with:
           lfs: false
           path: 'morpheus'
+          submodules: 'recursive'
 
       - name: Get AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@v1-node16
@@ -180,6 +183,9 @@ jobs:
         with:
           lfs: false
           path: 'morpheus'
+          # Fetch tags so that documentation builds for releases will report the version number correctly
+          fetch-tags: true
+          submodules: 'recursive'
 
       - name: Get AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@v1-node16
@@ -213,6 +219,7 @@ jobs:
           lfs: false
           path: 'morpheus'
           fetch-depth: 0
+          submodules: 'recursive'
 
       - name: Get AWS credentials using OIDC
         uses: aws-actions/configure-aws-credentials@v1-node16

CHANGELOG.md

@@ -14,6 +14,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 -->
+# Morpheus 24.03.02 (24 Apr 2024)
+
+## 🐛 Bug Fixes
+
+- Don't set pe_count for the C++ impl of the TritonInferenceStage ([#1640](https://github.com/nv-morpheus/Morpheus/pull/1640)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Fix vdb_upload runtime error ([#1643](https://github.com/nv-morpheus/Morpheus/pull/1643)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+## 📖 Documentation
+
+- Document current known issues in 24.03.02 ([#1656](https://github.com/nv-morpheus/Morpheus/pull/1656)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Fix documentation for building examples ([#1659](https://github.com/nv-morpheus/Morpheus/pull/1659)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Fix type-o in documentation ([#1662](https://github.com/nv-morpheus/Morpheus/pull/1662)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Fix mis-spelling of examples ([#1664](https://github.com/nv-morpheus/Morpheus/pull/1664)) [@dagardner-nv](https://github.com/dagardner-nv)
+
 # Morpheus 24.03.01 (10 Apr 2024)
 
 ## 🚨 Breaking Changes

ci/check_style.sh

@@ -16,7 +16,7 @@ rapids-dependency-file-generator \
   --file_key checks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
-rapids-mamba-retry env create --force -f env.yaml -n checks
+rapids-mamba-retry env create --yes -f env.yaml -n checks
 conda activate checks
 
 # Run pre-commit checks

ci/conda/recipes/run_conda_build.sh

@@ -19,7 +19,7 @@ NUMARGS=$#
 ARGS=$*
 
 function hasArg {
-    (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+   (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
 }
 
 function get_version() {
@@ -99,6 +99,14 @@ CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvid
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}pytorch")
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}defaults")
 
+if [[ ${NUMARGS} == 0 ]]; then
+   echo -e "${r}ERROR: No arguments were provided. Please provide at least one package to build. Available packages:${x}"
+   echo -e "${r}   morpheus${x}"
+   echo -e "${r}   pydebug${x}"
+   echo -e "${r}Exiting...${x}"
+   exit 12
+fi
+
 if hasArg morpheus; then
    # Set GIT_VERSION to set the project version inside of meta.yaml
    export GIT_VERSION="$(get_version)"
@@ -110,10 +118,10 @@ if hasArg morpheus; then
 fi
 
 if hasArg pydebug; then
-  export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2)
+   export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2)
 
-  echo "Running conda-build for python-dbg..."
-  set -x
-  conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ./ci/conda/recipes/python-dbg
-  set +x
+   echo "Running conda-build for python-dbg..."
+   set -x
+   conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ./ci/conda/recipes/python-dbg
+   set +x
 fi

ci/scripts/bootstrap_local_ci.sh

@@ -25,6 +25,8 @@ else
     git checkout ${GIT_BRANCH}
     git pull
     git checkout ${GIT_COMMIT}
+    git fetch --tags
+    git submodule update --init --recursive
 fi
 
 export MORPHEUS_ROOT=$(pwd)

ci/scripts/common.sh

@@ -73,7 +73,7 @@ function get_modified_files() {
    local GIT_DIFF_BASE=${GIT_DIFF_BASE:-$(get_merge_base)}
 
    # If invoked by a git-commit-hook, this will be populated
-   local result=( $(git diff ${GIT_DIFF_ARGS} $(get_merge_base) | grep -P ${1:-'.*'}) )
+   local result=( $(git diff ${GIT_DIFF_ARGS} ${GIT_DIFF_BASE} | grep -P ${1:-'.*'}) )
 
    local files=()
 

ci/scripts/github/build.sh

@@ -21,41 +21,33 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 rapids-dependency-file-generator \
   --output conda \
   --file_key build \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml"
 
-update_conda_env env.yaml
+update_conda_env "${WORKSPACE_TMP}/env.yaml"
 
 log_toolchain
 
-git submodule update --init --recursive
-
 CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_WHEEL=ON"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON"
-if [[ "${LOCAL_CI}" == "" ]]; then
-    CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)"
-fi
 
 rapids-logger "Configuring cmake for Morpheus with ${CMAKE_FLAGS}"
-cmake -B build -G Ninja ${CMAKE_FLAGS} .
+cmake ${CMAKE_FLAGS} .
 
 rapids-logger "Building Morpheus"
-cmake --build build --parallel ${PARALLEL_LEVEL}
+cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL}
 
-if [[ "${LOCAL_CI}" == "" ]]; then
-    rapids-logger "sccache usage for morpheus build:"
-    sccache --show-stats
-fi
+log_sccache_stats
 
 rapids-logger "Archiving results"
-tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" build/dist
+tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" ${BUILD_DIR}/dist
 
-MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/build/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \
+MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \
                 $(find ${MORPHEUS_ROOT}/examples -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
 tar cfj "${WORKSPACE_TMP}/morhpeus_libs.tar.bz" "${MORPHEUS_LIBS[@]}"
 
-CPP_TESTS=($(find ${MORPHEUS_ROOT}/build/morpheus/_lib/tests -name "*.x" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
+CPP_TESTS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/morpheus/_lib/tests -name "*.x" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
 tar cfj "${WORKSPACE_TMP}/cpp_tests.tar.bz" "${CPP_TESTS[@]}"
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}"

ci/scripts/github/checks.sh

@@ -21,42 +21,34 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 rapids-dependency-file-generator \
   --output conda \
   --file_key build \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml"
 
-update_conda_env env.yaml
+update_conda_env "${WORKSPACE_TMP}/env.yaml"
 
 log_toolchain
 
 cd ${MORPHEUS_ROOT}
 
+# Fetching the base branch will try methods that might fail, then fallback to one that does, set +e for this section
+set +e
 fetch_base_branch
-
-git submodule update --init --recursive
+set -e
 
 rapids-logger "Configuring cmake for Morpheus"
 CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF"
 export CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_INPLACE_BUILD=ON"
-if [[ "${LOCAL_CI}" == "" ]]; then
-    CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)"
-fi
 
-cmake -B build -G Ninja ${CMAKE_FLAGS} .
+cmake ${CMAKE_FLAGS} .
 
 rapids-logger "Building Morpheus"
-cmake --build build --parallel ${PARALLEL_LEVEL}
+cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL}
 
-if [[ "${LOCAL_CI}" == "" ]]; then
-    rapids-logger "sccache usage for source build:"
-    sccache --show-stats
-fi
+log_sccache_stats
 
 rapids-logger "Installing Morpheus"
 pip install ./
 
-# Setting this prevents loading of cudf since we don't have a GPU
-export MORPHEUS_IN_SPHINX_BUILD=1
-
 rapids-logger "Checking copyright headers"
 python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT}
 

ci/scripts/github/common.sh

@@ -61,7 +61,26 @@ export SCCACHE_REGION="us-east-2"
 export SCCACHE_IDLE_TIMEOUT=32768
 #export SCCACHE_LOG=debug
 
-export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_PYTHON_BUILD_STUBS=ON -DMORPHEUS_USE_CCACHE=ON"
+# Set the build flags
+export BUILD_DIR=${BUILD_DIR:-build}
+
+_FLAGS=()
+_FLAGS+=("-B" "${BUILD_DIR}")
+_FLAGS+=("-G" "Ninja")
+_FLAGS+=("-DCMAKE_MESSAGE_CONTEXT_SHOW=ON")
+_FLAGS+=("-DMORPHEUS_CUDA_ARCHITECTURES=RAPIDS")
+_FLAGS+=("-DMORPHEUS_USE_CONDA=ON")
+_FLAGS+=("-DMORPHEUS_USE_CCACHE=ON")
+_FLAGS+=("-DMORPHEUS_PYTHON_INPLACE_BUILD=OFF")
+_FLAGS+=("-DMORPHEUS_PYTHON_BUILD_STUBS=ON")
+_FLAGS+=("-DMORPHEUS_BUILD_BENCHMARKS=ON")
+_FLAGS+=("-DMORPHEUS_BUILD_EXAMPLES=ON")
+_FLAGS+=("-DMORPHEUS_BUILD_TESTS=ON")
+if [[ "${LOCAL_CI}" == "" ]]; then
+    _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)")
+fi
+export CMAKE_BUILD_ALL_FEATURES="${_FLAGS[@]}"
+unset _FLAGS
 
 if [[ ${MORPHEUS_SUPPORT_DOCA} == @(TRUE|ON) ]]; then
     export CMAKE_BUILD_ALL_FEATURES="${CMAKE_BUILD_ALL_FEATURES} -DMORPHEUS_SUPPORT_DOCA=ON"
@@ -116,8 +135,11 @@ function fetch_base_branch_gh_api() {
 
 function fetch_base_branch_local() {
     rapids-logger "Retrieving base branch from git"
-    git remote add upstream ${GIT_UPSTREAM_URL}
-    git fetch upstream --tags
+    if [[ "${USE_HOST_GIT}" == "0" ]]; then
+        git remote add upstream ${GIT_UPSTREAM_URL}
+        git fetch upstream --tags
+    fi
+
     source ${MORPHEUS_ROOT}/ci/scripts/common.sh
     export BASE_BRANCH=$(get_base_branch)
     export CHANGE_TARGET="upstream/${BASE_BRANCH}"
@@ -151,6 +173,13 @@ function log_toolchain() {
     sccache --version
 }
 
+function log_sccache_stats() {
+    if [[ "${LOCAL_CI}" == "" ]]; then
+        rapids-logger "sccache usage for morpheus build:"
+        sccache --show-stats
+    fi
+}
+
 function upload_artifact() {
     FILE_NAME=$1
     BASE_NAME=$(basename "${FILE_NAME}")