diff --git a/CMakeLists.txt b/CMakeLists.txt index a295b5ac5d..401a76ae5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,9 @@ option(BUILD_SHARED_LIBS "Default value for whether or not to build shared or st option(MORPHEUS_BUILD_BENCHMARKS "Whether or not to build benchmarks" OFF) option(MORPHEUS_BUILD_DOCS "Enable building of API documentation" OFF) option(MORPHEUS_BUILD_EXAMPLES "Whether or not to build examples" OFF) +option(MORPHEUS_BUILD_MORPHEUS_CORE "Whether or not to build morpheus_core" ON) option(MORPHEUS_BUILD_MORPHEUS_LLM "Whether or not to build morpheus_llm" OFF) +option(MORPHEUS_BUILD_MORPHEUS_DFP "Whether or not to build morpheus_dfp" OFF) option(MORPHEUS_BUILD_TESTS "Whether or not to build tests" OFF) option(MORPHEUS_ENABLE_DEBUG_INFO "Enable printing debug information" OFF) option(MORPHEUS_PYTHON_BUILD_STUBS "Whether or not to generated .pyi stub files for C++ Python modules. Disable to avoid requiring loading the NVIDIA GPU Driver during build" ON) @@ -51,6 +53,10 @@ if (MORPHEUS_BUILD_DOCS AND NOT MORPHEUS_BUILD_MORPHEUS_LLM) message(FATAL_ERROR "MORPHEUS_BUILD_MORPHEUS_LLM must be ON if MORPHEUS_BUILD_DOCS is ON") endif() +if (MORPHEUS_BUILD_DOCS AND NOT MORPHEUS_BUILD_MORPHEUS_DFP) + message(FATAL_ERROR "MORPHEUS_BUILD_MORPHEUS_DFP must be ON if MORPHEUS_BUILD_DOCS is ON") +endif() + # MRC CMake path and module extensions set(MORPHEUS_CMAKE_MODULE_PATH_EXTENSIONS "${CMAKE_CURRENT_SOURCE_DIR}/cmake" diff --git a/ci/conda/recipes/morpheus-core/meta.yaml b/ci/conda/recipes/morpheus-core/meta.yaml index f72004b95a..bd52d6558c 100644 --- a/ci/conda/recipes/morpheus-core/meta.yaml +++ b/ci/conda/recipes/morpheus-core/meta.yaml @@ -35,8 +35,6 @@ outputs: - CMAKE_CUDA_ARCHITECTURES - MORPHEUS_CACHE_DIR - MORPHEUS_PYTHON_BUILD_STUBS - - MORPHEUS_SUPPORT_DOCA - - MORPHEUS_BUILD_MORPHEUS_LLM - PARALLEL_LEVEL run_exports: - {{ pin_subpackage("morpheus-core", max_pin="x.x") }} diff --git a/ci/conda/recipes/morpheus-core/morpheus_core_build.sh b/ci/conda/recipes/morpheus-core/morpheus_core_build.sh index 5ef4920b9e..ecf2fadf2e 100644 --- a/ci/conda/recipes/morpheus-core/morpheus_core_build.sh +++ b/ci/conda/recipes/morpheus-core/morpheus_core_build.sh @@ -16,10 +16,6 @@ # It is assumed that this script is executed from the root of the repo directory by conda-build # (https://conda-forge.org/docs/maintainer/knowledge_base.html#using-cmake) -# Need to ensure this value is set before checking it in the if block -MORPHEUS_SUPPORT_DOCA=-OFF -MORPHEUS_BUILD_MORPHEUS_LLM=-OFF - # This will store all of the cmake args. Make sure to prepend args to allow # incoming values to overwrite them CMAKE_ARGS=${CMAKE_ARGS:-""} @@ -36,6 +32,14 @@ if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then mkdir -p ${MORPHEUS_CACHE_DIR} fi +# Enable core. Core is enabled by default and this is to just highlight that it is on +CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_CORE=ON ${CMAKE_ARGS}" + +# Disable dfp, llm and doca +CMAKE_ARGS="-DMORPHEUS_SUPPORT_DOCA=OFF ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_DFP=OFF ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=OFF ${CMAKE_ARGS}" + CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}" diff --git a/ci/conda/recipes/morpheus-dfp/conda_build_config.yaml b/ci/conda/recipes/morpheus-dfp/conda_build_config.yaml new file mode 100644 index 0000000000..4b051dc074 --- /dev/null +++ b/ci/conda/recipes/morpheus-dfp/conda_build_config.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +c_compiler_version: + - 11.2 + +cxx_compiler_version: + - 11.2 + +cuda_compiler: + - cuda-nvcc + +cuda_compiler_version: + - 12.1 + +python: + - 3.10 + +rapids_version: + - 24.02 diff --git a/ci/conda/recipes/morpheus-dfp/meta.yaml b/ci/conda/recipes/morpheus-dfp/meta.yaml new file mode 100644 index 0000000000..574827c53a --- /dev/null +++ b/ci/conda/recipes/morpheus-dfp/meta.yaml @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{% set version = environ.get('GIT_VERSION', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} + +package: + name: morpheus-dfp + version: {{ version }} + +source: + git_url: ../../../.. + +outputs: + + - name: morpheus-dfp + build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda_{{ cuda_compiler_version }}_py{{ python }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + # Note: Setting a default value here will overwrite the value from the + # shell! Defaults should be set in the run_conda_build.sh script + - CMAKE_CUDA_ARCHITECTURES + - MORPHEUS_CACHE_DIR + - MORPHEUS_PYTHON_BUILD_STUBS + - PARALLEL_LEVEL + run_exports: + - {{ pin_subpackage("morpheus-dfp", max_pin="x.x") }} + script: morpheus_dfp_build.sh + + requirements: + build: + - ccache + - cmake =3.27 + host: + - morpheus-core # TBD - version + - pip + - python {{ python }} + - scikit-build 0.17.6 + - versioneer-518 + run: + - morpheus-core # TBD - version + + test: + requires: + - pynvml + - pytest-asyncio + - pytest-benchmark + - pytest-cov + - pytest =7.4.4 + source_files: + - pyproject.toml + - scripts/fetch_data.py + imports: + - morpheus_dfp + commands: + - echo # make sure test requirements get installed + +about: + home: https://github.com/nv-morpheus/Morpheus + license: Apache-2.0 + license_family: Apache + license_file: LICENSE + summary: Morpheus Cybersecurity Digital Fingerprinting Library diff --git a/ci/conda/recipes/morpheus-dfp/morpheus_dfp_build.sh b/ci/conda/recipes/morpheus-dfp/morpheus_dfp_build.sh new file mode 100644 index 0000000000..8101741884 --- /dev/null +++ b/ci/conda/recipes/morpheus-dfp/morpheus_dfp_build.sh @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# It is assumed that this script is executed from the root of the repo directory by conda-build +# (https://conda-forge.org/docs/maintainer/knowledge_base.html#using-cmake) + +# This will store all of the cmake args. Make sure to prepend args to allow +# incoming values to overwrite them +CMAKE_ARGS=${CMAKE_ARGS:-""} + +export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..) +export USE_SCCACHE=${USE_SCCACHE:-""} + +# Check for some mrc environment variables. Append to front of args to allow users to overwrite them +if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then + # Set the cache variable, then set the Staging prefix to allow for host searching + CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}" + + # Double check that the cache dir has been created + mkdir -p ${MORPHEUS_CACHE_DIR} +fi + +# Enable DFP +CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_DFP=ON ${CMAKE_ARGS}" + +# Disable core, llm and doca +CMAKE_ARGS="-DMORPHEUS_SUPPORT_DOCA=OFF ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_CORE=OFF ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=OFF ${CMAKE_ARGS}" + +CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}" +CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_USE_CONDA=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS=-"ON"} ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_PYTHON_INPLACE_BUILD=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_WHEEL=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"RAPIDS"} ${CMAKE_ARGS}" +CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" +CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" # for pybind11 +CMAKE_ARGS="--log-level=VERBOSE ${CMAKE_ARGS}" + +if [[ "${USE_SCCACHE}" == "1" ]]; then + CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}" +fi + +echo "========Begin Env========" +env +echo "========End Env========" + +BUILD_DIR="build-conda" + +# Check if the build directory already exists. And if so, delete the +# CMakeCache.txt and CMakeFiles to ensure a clean configuration +if [[ -d "./${BUILD_DIR}" ]]; then + echo "Deleting old CMake files at ./${BUILD_DIR}" + rm -rf "./${BUILD_DIR}/CMakeCache.txt" + rm -rf "./${BUILD_DIR}/CMakeFiles" +fi + +# Run configure +cmake -B ${BUILD_DIR} \ + ${CMAKE_ARGS} \ + --log-level=verbose \ + . + +# Build the components +cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install + +# Install the mprpheus dfp python wheel components +${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus_dfp/dist/*.whl diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index 00b15fe488..fc55e54f30 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -36,7 +36,6 @@ outputs: - MORPHEUS_CACHE_DIR - MORPHEUS_PYTHON_BUILD_STUBS - MORPHEUS_SUPPORT_DOCA - - MORPHEUS_BUILD_MORPHEUS_LLM - PARALLEL_LEVEL run_exports: - {{ pin_subpackage("morpheus", max_pin="x.x") }} diff --git a/ci/conda/recipes/morpheus/morpheus_build.sh b/ci/conda/recipes/morpheus/morpheus_build.sh index 723559b4af..ad5771566c 100644 --- a/ci/conda/recipes/morpheus/morpheus_build.sh +++ b/ci/conda/recipes/morpheus/morpheus_build.sh @@ -18,7 +18,6 @@ # Need to ensure this value is set before checking it in the if block MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} -MORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-OFF} # This will store all of the cmake args. Make sure to prepend args to allow # incoming values to overwrite them @@ -45,9 +44,10 @@ if [[ ${MORPHEUS_SUPPORT_DOCA} == @(TRUE|ON) ]]; then echo "MORPHEUS_SUPPORT_DOCA is ON. Setting CMAKE_CUDA_ARCHITECTURES to supported values: '${CMAKE_CUDA_ARCHITECTURES}'" fi -if [[ ${MORPHEUS_BUILD_MORPHEUS_LLM} == @(TRUE|ON) ]]; then - CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=ON ${CMAKE_ARGS}" -fi +# enable all functional blocks +CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_CORE=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_DFP=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}" @@ -100,3 +100,4 @@ cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install # Install just the python wheel components ${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus/dist/*.whl ${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus_llm/dist/*.whl +${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus_dfp/dist/*.whl diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh index 3f96d70736..6900d8b993 100755 --- a/ci/conda/recipes/run_conda_build.sh +++ b/ci/conda/recipes/run_conda_build.sh @@ -127,6 +127,7 @@ if [[ ${NUMARGS} == 0 ]]; then echo -e "${r}ERROR: No arguments were provided. Please provide at least one package to build. Available packages:${x}" echo -e "${r} morpheus${x}" echo -e "${r} morpheus-core${x}" + echo -e "${r} morpheus-dfp${x}" echo -e "${r} pydebug${x}" echo -e "${r}Exiting...${x}" exit 12 @@ -134,7 +135,6 @@ fi if hasArg morpheus; then export MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} - export MORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-ON} # Set GIT_VERSION to set the project version inside of meta.yaml export GIT_VERSION="$(get_version)" @@ -145,8 +145,6 @@ if hasArg morpheus; then fi if hasArg morpheus-core; then - export MORPHEUS_SUPPORT_DOCA=-OFF - export MORPHEUS_BUILD_MORPHEUS_LLM=-OFF # Set GIT_VERSION to set the project version inside of meta.yaml export GIT_VERSION="$(get_version)" @@ -156,6 +154,16 @@ if hasArg morpheus-core; then set +x fi +if hasArg morpheus-dfp; then + # Set GIT_VERSION to set the project version inside of meta.yaml + export GIT_VERSION="$(get_version)" + + echo "Running conda-build for morpheus-dfp v${GIT_VERSION}..." + set -x + conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ci/conda/recipes/morpheus-dfp + set +x +fi + if hasArg pydebug; then export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2) diff --git a/ci/runner/Dockerfile b/ci/runner/Dockerfile index ed150d0657..40b035c402 100644 --- a/ci/runner/Dockerfile +++ b/ci/runner/Dockerfile @@ -60,7 +60,6 @@ RUN rapids-dependency-file-generator \ rm -rf /tmp/conda ENV MORPHEUS_SUPPORT_DOCA=ON -ENV MORPHEUS_BUILD_MORPHEUS_LLM=ON COPY ./.devcontainer/docker/optional_deps/doca.sh /tmp/doca/ diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh index 7babe4a40a..8f75893967 100755 --- a/ci/scripts/github/build.sh +++ b/ci/scripts/github/build.sh @@ -42,7 +42,7 @@ cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL} log_sccache_stats rapids-logger "Archiving results" -tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" ${BUILD_DIR}/python/morpheus/dist ${BUILD_DIR}/python/morpheus_llm/dist +tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" ${BUILD_DIR}/python/morpheus/dist ${BUILD_DIR}/python/morpheus_llm/dist ${BUILD_DIR}/python/morpheus_dfp/dist MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/python/morpheus/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \ $(find ${MORPHEUS_ROOT}/${BUILD_DIR}/python/morpheus_llm/morpheus_llm/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \ diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh index 6b6e579feb..a00afe718f 100755 --- a/ci/scripts/github/checks.sh +++ b/ci/scripts/github/checks.sh @@ -50,6 +50,7 @@ log_sccache_stats rapids-logger "Installing Morpheus" pip install ./python/morpheus pip install ./python/morpheus_llm +pip install ./python/morpheus_dfp rapids-logger "Checking copyright headers" python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT} diff --git a/ci/scripts/github/cmake_all.sh b/ci/scripts/github/cmake_all.sh index 86b0d65dcd..0e378a505a 100644 --- a/ci/scripts/github/cmake_all.sh +++ b/ci/scripts/github/cmake_all.sh @@ -27,6 +27,7 @@ _FLAGS+=("-DMORPHEUS_BUILD_BENCHMARKS=ON") _FLAGS+=("-DMORPHEUS_BUILD_EXAMPLES=ON") _FLAGS+=("-DMORPHEUS_BUILD_TESTS=ON") _FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_LLM=ON") +_FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_DFP=ON") if [[ "${LOCAL_CI}" == "" ]]; then _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)") fi diff --git a/ci/scripts/github/cmake_core.sh b/ci/scripts/github/cmake_core.sh index 20d250e45c..7e1dfe7e06 100644 --- a/ci/scripts/github/cmake_core.sh +++ b/ci/scripts/github/cmake_core.sh @@ -27,6 +27,7 @@ _FLAGS+=("-DMORPHEUS_BUILD_BENCHMARKS=OFF") _FLAGS+=("-DMORPHEUS_BUILD_EXAMPLES=OFF") _FLAGS+=("-DMORPHEUS_BUILD_TESTS=OFF") _FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_LLM=OFF") +_FLAGS+=("-DMORPHEUS_BUILD_MORPHEUS_DFP=OFF") _FLAGS+=("-DMORPHEUS_SUPPORT_DOCA=OFF") if [[ "${LOCAL_CI}" == "" ]]; then _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)") diff --git a/ci/scripts/github/conda_dfp.sh b/ci/scripts/github/conda_dfp.sh new file mode 100755 index 0000000000..3f98fe998a --- /dev/null +++ b/ci/scripts/github/conda_dfp.sh @@ -0,0 +1,61 @@ +#!/usr/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +CI_SCRIPT_ARGS="$@" +source ${WORKSPACE}/ci/scripts/github/common.sh + +cd ${MORPHEUS_ROOT} + +fetch_base_branch + +# Its important that we are in the base environment for the build +rapids-logger "Activating Base Conda Environment" + +# Deactivate any extra environments (There can be a few on the stack) +while [[ "${CONDA_SHLVL:-0}" -gt 1 ]]; do + echo "Deactivating conda environment ${CONDA_DEFAULT_ENV}" + conda deactivate +done + +# Ensure at least base is activated +if [[ "${CONDA_DEFAULT_ENV}" != "base" ]]; then + echo "Activating base conda environment" + conda activate base +fi + +# Print the info just to be sure base is active +conda info + +rapids-logger "Building Conda Package morpheus-dfp" + +# Run the conda build, and upload to conda forge if requested +export MORPHEUS_PYTHON_BUILD_STUBS=OFF +export CONDA_ARGS="--skip-existing" +${MORPHEUS_ROOT}/ci/conda/recipes/run_conda_build.sh morpheus-dfp "${CI_SCRIPT_ARGS}" + +# If we didn't receive the upload argument, upload the artifact to S3 +if [[ " ${CI_SCRIPT_ARGS} " =~ " upload " ]]; then + rapids-logger "Building Conda Package morpheus-dfp... Done" +else + # if we didn't receive the upload argument, we can still upload the artifact to S3 + tar cfj "${WORKSPACE_TMP}/conda_morpheus_dfp.tar.bz" "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" + ls -lh ${WORKSPACE_TMP}/ + + rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" + upload_artifact "${WORKSPACE_TMP}/conda_morpheus_dfp.tar.bz" +fi diff --git a/ci/scripts/github/docs.sh b/ci/scripts/github/docs.sh index a0e2057fe8..441c8495b0 100755 --- a/ci/scripts/github/docs.sh +++ b/ci/scripts/github/docs.sh @@ -32,6 +32,7 @@ tar xf "${WORKSPACE_TMP}/wheel.tar.bz" pip install ${MORPHEUS_ROOT}/${BUILD_DIR}/python/morpheus/dist/*.whl pip install ${MORPHEUS_ROOT}/${BUILD_DIR}/python/morpheus_llm/dist/*.whl +pip install ${MORPHEUS_ROOT}/${BUILD_DIR}/python/morpheus_dfp/dist/*.whl rapids-logger "Pulling LFS assets" cd ${MORPHEUS_ROOT} diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh index 8f7df827da..ff1a717733 100755 --- a/ci/scripts/run_ci_local.sh +++ b/ci/scripts/run_ci_local.sh @@ -21,13 +21,13 @@ case "$1" in STAGES=("bash") ;; "all" ) - STAGES=("checks" "build" "docs" "test" "conda_core" "conda") + STAGES=("checks" "build" "docs" "test" "conda_core" "conda_dfp" "conda") ;; - "checks" | "build" | "docs" | "test" | "conda_core" | "conda" | "bash" ) + "checks" | "build" | "docs" | "test" | "conda_core" | "conda_dfp" | "conda" | "bash" ) STAGES=("$1") ;; * ) - echo "Error: Invalid argument \"$1\" provided. Expected values: \"all\", \"checks\", \"build\", \"docs\", \"test\", \"conda_core\", \"conda\", or \"bash\"" + echo "Error: Invalid argument \"$1\" provided. Expected values: \"all\", \"checks\", \"build\", \"docs\", \"test\", \"conda_core\", \"conda_dfp\", \"conda\", or \"bash\"" exit 1 ;; esac diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 2b59d30593..284be4bd77 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -75,6 +75,7 @@ if(MORPHEUS_BUILD_TESTS) ) endif() +if(MORPHEUS_BUILD_MORPHEUS_CORE) # cccl -- get an explicit cccl build, matx tries to pull a tag that doesn't exist. # ========= morpheus_utils_configure_cccl() @@ -106,5 +107,6 @@ morpheus_utils_configure_cudf() # Triton-client # ===== morpheus_utils_configure_tritonclient() +endif() list(POP_BACK CMAKE_MESSAGE_CONTEXT) diff --git a/docker/build_container.sh b/docker/build_container.sh index ca7f7cad78..85877583f7 100755 --- a/docker/build_container.sh +++ b/docker/build_container.sh @@ -36,6 +36,7 @@ LINUX_DISTRO=${LINUX_DISTRO:-ubuntu} LINUX_VER=${LINUX_VER:-22.04} MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-"OFF"} MORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-"ON"} +MORPHEUS_BUILD_MORPHEUS_DFP=${MORPHEUS_BUILD_MORPHEUS_DFP:-"ON"} PYTHON_VER=${PYTHON_VER:-3.10} # Determine the relative path from $PWD to $MORPHEUS_ROOT @@ -53,6 +54,7 @@ DOCKER_ARGS="${DOCKER_ARGS} --build-arg LINUX_VER=${LINUX_VER}" DOCKER_ARGS="${DOCKER_ARGS} --build-arg MORPHEUS_ROOT_HOST=${MORPHEUS_ROOT_HOST}" DOCKER_ARGS="${DOCKER_ARGS} --build-arg MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA}" DOCKER_ARGS="${DOCKER_ARGS} --build-arg MORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM}" +DOCKER_ARGS="${DOCKER_ARGS} --build-arg MORPHEUS_BUILD_MORPHEUS_DFP=${MORPHEUS_BUILD_MORPHEUS_DFP}" DOCKER_ARGS="${DOCKER_ARGS} --build-arg PYTHON_VER=${PYTHON_VER}" DOCKER_ARGS="${DOCKER_ARGS} --network=host" diff --git a/docker/run_container_release.sh b/docker/run_container_release.sh index 42575d394e..dce2132b1a 100755 --- a/docker/run_container_release.sh +++ b/docker/run_container_release.sh @@ -29,6 +29,7 @@ pushd ${SCRIPT_DIR} &> /dev/null MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} MORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-ON} +MORPHEUS_BUILD_MORPHEUS_DFP=${MORPHEUS_BUILD_MORPHEUS_DFP:-ON} DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"nvcr.io/nvidia/morpheus/morpheus"} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"$(git describe --tags --abbrev=0)-runtime"} diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 38439e9d43..d4b3ab9acb 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -199,6 +199,7 @@ git submodule update --init --recursive ```bash pip install -e ${MORPHEUS_ROOT}/python/morpheus pip install -e ${MORPHEUS_ROOT}/python/morpheus_llm + pip install -e ${MORPHEUS_ROOT}/python/morpheus_dfp ``` Once Morpheus has been built, it can be installed into the current virtual environment. 1. Test the build (Note: some tests will be skipped)\ diff --git a/docs/source/py_api.rst b/docs/source/py_api.rst index e4aa991db2..63fcb5c447 100644 --- a/docs/source/py_api.rst +++ b/docs/source/py_api.rst @@ -23,3 +23,4 @@ Python API morpheus morpheus_llm + morpheus_dfp diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index e94eadfe80..6203dbd2f1 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -15,12 +15,16 @@ list(APPEND CMAKE_MESSAGE_CONTEXT "python") +if (MORPHEUS_BUILD_MORPHEUS_CORE) add_subdirectory(morpheus) +endif() if (MORPHEUS_BUILD_MORPHEUS_LLM) add_subdirectory(morpheus_llm) endif() +if (MORPHEUS_BUILD_MORPHEUS_DFP) add_subdirectory(morpheus_dfp) +endif() list(POP_BACK CMAKE_MESSAGE_CONTEXT) diff --git a/scripts/compile.sh b/scripts/compile.sh index cf8c628515..cb0359b95c 100755 --- a/scripts/compile.sh +++ b/scripts/compile.sh @@ -34,6 +34,7 @@ cmake -S . -B ${BUILD_DIR} -GNinja \ -DMORPHEUS_USE_CONDA=${MORPHEUS_USE_CONDA:-ON} \ -DMORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} \ -DMORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-ON} \ + -DMORPHEUS_BUILD_MORPHEUS_DFP=${MORPHEUS_BUILD_MORPHEUS_DFP:-ON} \ ${INSTALL_PREFIX:+-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}} \ ${CMAKE_ARGS:+${CMAKE_ARGS}} \ ${CMAKE_CONFIGURE_EXTRA_ARGS:+${CMAKE_CONFIGURE_EXTRA_ARGS}}