diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index 039920e272..10c086c551 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -63,6 +63,7 @@ outputs: - mrc {{ minor_version }} - pip - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF + - pyarrow-hotfix # CVE-2023-47248 - pybind11-stubgen 0.10.5 - python {{ python }} - rapidjson 1.1 @@ -92,6 +93,7 @@ outputs: - pandas 1.3.* - pluggy 1.0.* - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF + - pyarrow-hotfix # CVE-2023-47248 - python - python-confluent-kafka 1.9.2 - pytorch 2.0.1 @@ -99,7 +101,7 @@ outputs: - requests=2.31 - requests-cache=1.1 - scikit-learn 1.2.2.* - - sqlalchemy <=2.0 # 2.0 is incompatible with pandas=1.3 + - sqlalchemy <2.0 # 2.0 is incompatible with pandas=1.3 - tqdm 4.* - tritonclient 2.26.* - typing_utils 0.1.* diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh index 3a52e10190..68fab15ce8 100755 --- a/ci/conda/recipes/run_conda_build.sh +++ b/ci/conda/recipes/run_conda_build.sh @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -48,6 +48,7 @@ export CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES:-"RAPIDS"} export MORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS:-"ON"} export MORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR:-"${MORPHEUS_ROOT}/.cache"} export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)} +export MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} # Set CONDA_CHANNEL_ALIAS to mimic the conda config channel_alias property during the build CONDA_CHANNEL_ALIAS=${CONDA_CHANNEL_ALIAS:-""} diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 95de177e43..705ed2724b 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -19,7 +19,6 @@ dependencies: - breathe=4.34.0 - ccache>=3.7 - clangdev=16 -- click >=8 - click>=8 - cmake=3.25 - cuda-cudart-dev=11.8 @@ -76,13 +75,14 @@ dependencies: - pluggy=1.0 - pre-commit - protobuf=4.21.* +- pyarrow-hotfix - pybind11-stubgen=0.10 - pylint>=2.17.4,<2.18 - pypdf=3.16 -- pytest - pytest-asyncio - pytest-benchmark>=4.0 - pytest-cov +- pytest=7.4 - python-confluent-kafka=1.9.2 - python-dotenv=1.0 - python-graphviz @@ -97,6 +97,7 @@ dependencies: - sphinx - sphinx_rtd_theme - sqlalchemy<2.0 +- sysroot_linux-64=2.17 - tqdm=4 - transformers=4.30.2 - tritonclient=2.26 @@ -113,7 +114,6 @@ dependencies: - grpcio-status==1.58 - milvus==2.3.2 - nemollm - - pyarrow_hotfix - pymilvus==2.3.2 - pytest-kafka==0.6.0 name: all_cuda-118_arch-x86_64_py-310 diff --git a/conda/environments/dev_cuda-118_arch-x86_64.yaml b/conda/environments/dev_cuda-118_arch-x86_64.yaml index 3552134802..14b3fb7b1e 100644 --- a/conda/environments/dev_cuda-118_arch-x86_64.yaml +++ b/conda/environments/dev_cuda-118_arch-x86_64.yaml @@ -17,7 +17,6 @@ dependencies: - breathe=4.34.0 - ccache>=3.7 - clangdev=16 -- click >=8 - click>=8 - cmake=3.25 - cuda-cudart-dev=11.8 @@ -61,12 +60,13 @@ dependencies: - pluggy=1.0 - pre-commit - protobuf=4.21.* +- pyarrow-hotfix - pybind11-stubgen=0.10 - pylint>=2.17.4,<2.18 -- pytest - pytest-asyncio - pytest-benchmark>=4.0 - pytest-cov +- pytest=7.4 - python-confluent-kafka=1.9.2 - python-graphviz - pytorch-cuda @@ -78,6 +78,7 @@ dependencies: - sphinx - sphinx_rtd_theme - sqlalchemy<2.0 +- sysroot_linux-64=2.17 - tqdm=4 - tritonclient=2.26 - typing_utils=0.1 @@ -89,7 +90,6 @@ dependencies: - pip: - databricks-connect - milvus==2.3.2 - - pyarrow_hotfix - pymilvus==2.3.2 - pytest-kafka==0.6.0 name: dev_cuda-118_arch-x86_64_py-310 diff --git a/dependencies.yaml b/dependencies.yaml index ea6ecf5fa5..0290f58ff7 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -30,7 +30,6 @@ files: - test_python_morpheus - docs - cudatoolkit - - runtime - checks dev: @@ -47,9 +46,18 @@ files: - test_python_morpheus - docs - cudatoolkit - - runtime - checks + examples: + output: conda + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - examples + - runtime + - cudatoolkit + build: output: none matrix: @@ -143,6 +151,7 @@ dependencies: - pybind11-stubgen=0.10 - rapidjson=1.1.0 - scikit-build=0.17.1 + - sysroot_linux-64=2.17 - tritonclient=2.26 # Required by NvTabular, force the version, so we get protobufs compatible with 4.21 - ucx=1.14 @@ -185,7 +194,7 @@ dependencies: - sphinx_rtd_theme - numpydoc=1.4 - pluggy=1.0 - - &click click >=8 + - &click click>=8 benchmark_cpp: common: @@ -245,6 +254,7 @@ dependencies: - nb_conda_kernels - numpydoc=1.4 - nvtabular=23.06 + - pyarrow-hotfix - python-confluent-kafka=1.9.2 - python-graphviz - pytorch-cuda @@ -260,7 +270,6 @@ dependencies: - pip: - databricks-connect - milvus==2.3.2 - - pyarrow_hotfix - pymilvus==2.3.2 test_python_morpheus: @@ -268,7 +277,7 @@ dependencies: - output_types: [conda] packages: - nodejs=18.* - - pytest + - pytest=7.4 # 8.0 appears to be incompatible with pytest-asyncio - pytest-asyncio - pytest-benchmark>=4.0 - pytest-cov diff --git a/docker/build_conda_packages.sh b/docker/build_conda_packages.sh index 19a040575d..43e0b1bbba 100755 --- a/docker/build_conda_packages.sh +++ b/docker/build_conda_packages.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,6 +36,7 @@ CUR_GID=$(id -g ${LOGNAME}) MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)} mkdir -p ${MORPHEUS_ROOT}/.cache/ccache mkdir -p ${MORPHEUS_ROOT}/.cache/cpm +mkdir -p ${MORPHEUS_ROOT}/.conda-bld echo "Building container" # Call the build script to get a container ready to build conda packages diff --git a/docker/conda/environments/cuda11.8_dev.yml b/docker/conda/environments/cuda11.8_dev.yml index 0527e4277b..8a1b313723 100644 --- a/docker/conda/environments/cuda11.8_dev.yml +++ b/docker/conda/environments/cuda11.8_dev.yml @@ -83,10 +83,11 @@ dependencies: - pluggy=1.0 - protobuf=4.21.* - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF + - pyarrow-hotfix # CVE-2023-47248. See morpheus/__init__.py for more details - pybind11-stubgen=0.10.5 - pydot - pylint>=2.17.4,<2.18 # 2.17.4 contains a fix for toml support - - pytest + - pytest=7.4 - pytest-asyncio - pytest-benchmark>=4.0 - pytest-cov @@ -124,6 +125,5 @@ dependencies: # Add additional dev dependencies here - databricks-connect - milvus==2.3.2 - - pyarrow_hotfix # CVE-2023-47248. See morpheus/__init__.py for more details - pymilvus==2.3.2 - pytest-kafka==0.6.0 diff --git a/examples/abp_nvsmi_detection/README.md b/examples/abp_nvsmi_detection/README.md index ce63ec15c0..e92e5f0ed6 100644 --- a/examples/abp_nvsmi_detection/README.md +++ b/examples/abp_nvsmi_detection/README.md @@ -1,5 +1,5 @@