morpheus_dfp conda recipe and CI updates

1. Add a build flag to disable compiling morpheus-core
2. Add a build flag for morpheus_dfp
3. Add morpheus_dfp conda recipe
4. Include morpheus_dfp package in docs

Cleanup:
1. Avoid enabling morpheus extended package flags in the runner container.
Instead set them via the build scripts. This is mainly to avoid the
confusion in managing the flags.
2. Simply passing build flags to the conda recipes. This was happening
in multiple ways making future maintanence challenging.

CMakeLists.txt

@@ -22,7 +22,9 @@ option(BUILD_SHARED_LIBS "Default value for whether or not to build shared or st
 option(MORPHEUS_BUILD_BENCHMARKS "Whether or not to build benchmarks" OFF)
 option(MORPHEUS_BUILD_DOCS "Enable building of API documentation" OFF)
 option(MORPHEUS_BUILD_EXAMPLES "Whether or not to build examples" OFF)
+option(MORPHEUS_BUILD_MORPHEUS_CORE "Whether or not to build morpheus_core" ON)
 option(MORPHEUS_BUILD_MORPHEUS_LLM "Whether or not to build morpheus_llm" OFF)
+option(MORPHEUS_BUILD_MORPHEUS_DFP "Whether or not to build morpheus_dfp" OFF)
 option(MORPHEUS_BUILD_TESTS "Whether or not to build tests" OFF)
 option(MORPHEUS_ENABLE_DEBUG_INFO "Enable printing debug information" OFF)
 option(MORPHEUS_PYTHON_BUILD_STUBS "Whether or not to generated .pyi stub files for C++ Python modules. Disable to avoid requiring loading the NVIDIA GPU Driver during build" ON)
@@ -51,6 +53,10 @@ if (MORPHEUS_BUILD_DOCS AND NOT MORPHEUS_BUILD_MORPHEUS_LLM)
   message(FATAL_ERROR "MORPHEUS_BUILD_MORPHEUS_LLM must be ON if MORPHEUS_BUILD_DOCS is ON")
 endif()
 
+if (MORPHEUS_BUILD_DOCS AND NOT MORPHEUS_BUILD_MORPHEUS_DFP)
+  message(FATAL_ERROR "MORPHEUS_BUILD_MORPHEUS_DFP must be ON if MORPHEUS_BUILD_DOCS is ON")
+endif()
+
 # MRC CMake path and module extensions
 set(MORPHEUS_CMAKE_MODULE_PATH_EXTENSIONS
   "${CMAKE_CURRENT_SOURCE_DIR}/cmake"

ci/conda/recipes/morpheus-core/meta.yaml

@@ -35,8 +35,6 @@ outputs:
         - CMAKE_CUDA_ARCHITECTURES
         - MORPHEUS_CACHE_DIR
         - MORPHEUS_PYTHON_BUILD_STUBS
-        - MORPHEUS_SUPPORT_DOCA
-        - MORPHEUS_BUILD_MORPHEUS_LLM
         - PARALLEL_LEVEL
       run_exports:
         - {{ pin_subpackage("morpheus-core", max_pin="x.x") }}

ci/conda/recipes/morpheus-core/morpheus_core_build.sh

@@ -16,10 +16,6 @@
 # It is assumed that this script is executed from the root of the repo directory by conda-build
 # (https://conda-forge.org/docs/maintainer/knowledge_base.html#using-cmake)
 
-# Need to ensure this value is set before checking it in the if block
-MORPHEUS_SUPPORT_DOCA=-OFF
-MORPHEUS_BUILD_MORPHEUS_LLM=-OFF
-
 # This will store all of the cmake args. Make sure to prepend args to allow
 # incoming values to overwrite them
 CMAKE_ARGS=${CMAKE_ARGS:-""}
@@ -36,6 +32,14 @@ if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
    mkdir -p ${MORPHEUS_CACHE_DIR}
 fi
 
+# Enable core. Core is enabled by default and this is to just highlight that it is on
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_CORE=ON ${CMAKE_ARGS}"
+
+# Disable dfp, llm and doca
+CMAKE_ARGS="-DMORPHEUS_SUPPORT_DOCA=OFF ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_DFP=OFF ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=OFF ${CMAKE_ARGS}"
+
 CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"

ci/conda/recipes/morpheus-dfp/conda_build_config.yaml

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+c_compiler_version:
+  - 11.2
+
+cxx_compiler_version:
+  - 11.2
+
+cuda_compiler:
+  - cuda-nvcc
+
+cuda_compiler_version:
+  - 12.1
+
+python:
+  - 3.10
+
+rapids_version:
+  - 24.02

ci/conda/recipes/morpheus-dfp/meta.yaml

@@ -0,0 +1,75 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{% set version = environ.get('GIT_VERSION', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
+
+package:
+  name: morpheus-dfp
+  version: {{ version }}
+
+source:
+  git_url: ../../../..
+
+outputs:
+
+  - name: morpheus-dfp
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda_{{ cuda_compiler_version }}_py{{ python }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+      script_env:
+        # Note: Setting a default value here will overwrite the value from the
+        # shell! Defaults should be set in the run_conda_build.sh script
+        - CMAKE_CUDA_ARCHITECTURES
+        - MORPHEUS_CACHE_DIR
+        - MORPHEUS_PYTHON_BUILD_STUBS
+        - PARALLEL_LEVEL
+      run_exports:
+        - {{ pin_subpackage("morpheus-dfp", max_pin="x.x") }}
+    script: morpheus_dfp_build.sh
+
+    requirements:
+      build:
+        - ccache
+        - cmake =3.27
+      host:
+        - morpheus-core # TBD - version
+        - pip
+        - python {{ python }}
+        - scikit-build 0.17.6
+        - versioneer-518
+      run:
+        - morpheus-core # TBD - version
+
+    test:
+      requires:
+        - pynvml
+        - pytest-asyncio
+        - pytest-benchmark
+        - pytest-cov
+        - pytest =7.4.4
+      source_files:
+        - pyproject.toml
+        - scripts/fetch_data.py
+      imports:
+        - morpheus_dfp
+      commands:
+          - echo # make sure test requirements get installed
+
+about:
+  home: https://github.com/nv-morpheus/Morpheus
+  license: Apache-2.0
+  license_family: Apache
+  license_file: LICENSE
+  summary: Morpheus Cybersecurity Digital Fingerprinting Library

ci/conda/recipes/morpheus-dfp/morpheus_dfp_build.sh

@@ -0,0 +1,86 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# It is assumed that this script is executed from the root of the repo directory by conda-build
+# (https://conda-forge.org/docs/maintainer/knowledge_base.html#using-cmake)
+
+# This will store all of the cmake args. Make sure to prepend args to allow
+# incoming values to overwrite them
+CMAKE_ARGS=${CMAKE_ARGS:-""}
+
+export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
+export USE_SCCACHE=${USE_SCCACHE:-""}
+
+# Check for some mrc environment variables. Append to front of args to allow users to overwrite them
+if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
+   # Set the cache variable, then set the Staging prefix to allow for host searching
+   CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"
+
+   # Double check that the cache dir has been created
+   mkdir -p ${MORPHEUS_CACHE_DIR}
+fi
+
+# Enable DFP
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_DFP=ON ${CMAKE_ARGS}"
+
+# Disable core, llm and doca
+CMAKE_ARGS="-DMORPHEUS_SUPPORT_DOCA=OFF ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_CORE=OFF ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=OFF ${CMAKE_ARGS}"
+
+CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
+CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_USE_CONDA=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS=-"ON"} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_INPLACE_BUILD=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_WHEEL=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"RAPIDS"} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" # for pybind11
+CMAKE_ARGS="--log-level=VERBOSE ${CMAKE_ARGS}"
+
+if [[ "${USE_SCCACHE}" == "1" ]]; then
+   CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}"
+fi
+
+echo "========Begin Env========"
+env
+echo "========End Env========"
+
+BUILD_DIR="build-conda"
+
+# Check if the build directory already exists. And if so, delete the
+# CMakeCache.txt and CMakeFiles to ensure a clean configuration
+if [[ -d "./${BUILD_DIR}" ]]; then
+   echo "Deleting old CMake files at ./${BUILD_DIR}"
+   rm -rf "./${BUILD_DIR}/CMakeCache.txt"
+   rm -rf "./${BUILD_DIR}/CMakeFiles"
+fi
+
+# Run configure
+cmake -B ${BUILD_DIR} \
+   ${CMAKE_ARGS} \
+   --log-level=verbose \
+   .
+
+# Build the components
+cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
+
+# Install the mprpheus dfp python wheel components
+${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus_dfp/dist/*.whl

ci/conda/recipes/morpheus/meta.yaml

@@ -36,7 +36,6 @@ outputs:
         - MORPHEUS_CACHE_DIR
         - MORPHEUS_PYTHON_BUILD_STUBS
         - MORPHEUS_SUPPORT_DOCA
-        - MORPHEUS_BUILD_MORPHEUS_LLM
         - PARALLEL_LEVEL
       run_exports:
         - {{ pin_subpackage("morpheus", max_pin="x.x") }}

ci/conda/recipes/morpheus/morpheus_build.sh

@@ -18,7 +18,6 @@
 
 # Need to ensure this value is set before checking it in the if block
 MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF}
-MORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-OFF}
 
 # This will store all of the cmake args. Make sure to prepend args to allow
 # incoming values to overwrite them
@@ -45,9 +44,10 @@ if [[ ${MORPHEUS_SUPPORT_DOCA} == @(TRUE|ON) ]]; then
    echo "MORPHEUS_SUPPORT_DOCA is ON. Setting CMAKE_CUDA_ARCHITECTURES to supported values: '${CMAKE_CUDA_ARCHITECTURES}'"
 fi
 
-if [[ ${MORPHEUS_BUILD_MORPHEUS_LLM} == @(TRUE|ON) ]]; then
-   CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=ON ${CMAKE_ARGS}"
-fi
+# enable all functional blocks
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_CORE=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_DFP=ON ${CMAKE_ARGS}"
 
 CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
@@ -100,3 +100,4 @@ cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
 # Install just the python wheel components
 ${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus/dist/*.whl
 ${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus_llm/dist/*.whl
+${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus_dfp/dist/*.whl

ci/conda/recipes/run_conda_build.sh

@@ -127,14 +127,14 @@ if [[ ${NUMARGS} == 0 ]]; then
    echo -e "${r}ERROR: No arguments were provided. Please provide at least one package to build. Available packages:${x}"
    echo -e "${r}   morpheus${x}"
    echo -e "${r}   morpheus-core${x}"
+   echo -e "${r}   morpheus-dfp${x}"
    echo -e "${r}   pydebug${x}"
    echo -e "${r}Exiting...${x}"
    exit 12
 fi
 
 if hasArg morpheus; then
    export MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF}
-   export MORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-ON}
    # Set GIT_VERSION to set the project version inside of meta.yaml
    export GIT_VERSION="$(get_version)"
 
@@ -145,8 +145,6 @@ if hasArg morpheus; then
 fi
 
 if hasArg morpheus-core; then
-   export MORPHEUS_SUPPORT_DOCA=-OFF
-   export MORPHEUS_BUILD_MORPHEUS_LLM=-OFF
    # Set GIT_VERSION to set the project version inside of meta.yaml
    export GIT_VERSION="$(get_version)"
 
@@ -156,6 +154,16 @@ if hasArg morpheus-core; then
    set +x
 fi
 
+if hasArg morpheus-dfp; then
+   # Set GIT_VERSION to set the project version inside of meta.yaml
+   export GIT_VERSION="$(get_version)"
+
+   echo "Running conda-build for morpheus-dfp v${GIT_VERSION}..."
+   set -x
+   conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ci/conda/recipes/morpheus-dfp
+   set +x
+fi
+
 if hasArg pydebug; then
    export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2)
 

ci/runner/Dockerfile

@@ -60,7 +60,6 @@ RUN rapids-dependency-file-generator \
     rm -rf /tmp/conda
 
 ENV MORPHEUS_SUPPORT_DOCA=ON
-ENV MORPHEUS_BUILD_MORPHEUS_LLM=ON
 
 COPY ./.devcontainer/docker/optional_deps/doca.sh /tmp/doca/
 

ci/scripts/github/build.sh

@@ -42,7 +42,7 @@ cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL}
 log_sccache_stats
 
 rapids-logger "Archiving results"
-tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" ${BUILD_DIR}/python/morpheus/dist ${BUILD_DIR}/python/morpheus_llm/dist
+tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" ${BUILD_DIR}/python/morpheus/dist ${BUILD_DIR}/python/morpheus_llm/dist ${BUILD_DIR}/python/morpheus_dfp/dist
 
 MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/python/morpheus/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \
                 $(find ${MORPHEUS_ROOT}/${BUILD_DIR}/python/morpheus_llm/morpheus_llm/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \

ci/scripts/github/checks.sh

@@ -50,6 +50,7 @@ log_sccache_stats
 rapids-logger "Installing Morpheus"
 pip install ./python/morpheus
 pip install ./python/morpheus_llm
+pip install ./python/morpheus_dfp
 
 rapids-logger "Checking copyright headers"
 python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT}

ci/scripts/github/cmake_all.sh

@@ -27,6 +27,7 @@ _FLAGS+=("-DMORPHEUS_BUILD_BENCHMARKS=ON")
 _FLAGS+=("-DMORPHEUS_BUILD_EXAMPLES=ON")
 _FLAGS+=("-DMORPHEUS_BUILD_TESTS=ON")
 _FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_LLM=ON")
+_FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_DFP=ON")
 if [[ "${LOCAL_CI}" == "" ]]; then
     _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)")
 fi

ci/scripts/github/cmake_core.sh

@@ -27,6 +27,7 @@ _FLAGS+=("-DMORPHEUS_BUILD_BENCHMARKS=OFF")
 _FLAGS+=("-DMORPHEUS_BUILD_EXAMPLES=OFF")
 _FLAGS+=("-DMORPHEUS_BUILD_TESTS=OFF")
 _FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_LLM=OFF")
+_FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_DFP=OFF")
 _FLAGS+=("-DMORPHEUS_SUPPORT_DOCA=OFF")
 if [[ "${LOCAL_CI}" == "" ]]; then
     _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)")