From 41c576adaf8b0350e31838d8b807a524d6922856 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 4 Dec 2023 18:27:42 +0000 Subject: [PATCH 01/19] reoder channels and set cuda-python version --- dependencies.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 4cec9440ca..97ac66c53b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -28,13 +28,13 @@ files: - cudatoolkit channels: + - conda-forge - rapidsai - nvidia/label/cuda-11.8.0 - nvidia - rapidsai-nightly - nvidia/label/dev - pytorch - - conda-forge dependencies: @@ -108,5 +108,6 @@ dependencies: - cuda-cudart-dev=11.8 - cuda-nvml-dev=11.8 - cuda-nvrtc-dev=11.8 + - cuda-python=11.8.2 - cuda-tools=11.8 - cuda-version=11.8 From 00d97cebe7f84aca43463116baf46426e9f3bb58 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 4 Dec 2023 20:25:46 +0000 Subject: [PATCH 02/19] add pyarrow_hotfix to dependencies, rename "python" conda package to morpheus-python-dbg --- ci/conda/recipes/python-dbg/meta.yaml | 2 +- dependencies.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/conda/recipes/python-dbg/meta.yaml b/ci/conda/recipes/python-dbg/meta.yaml index 3bf036a532..fe56414f5d 100644 --- a/ci/conda/recipes/python-dbg/meta.yaml +++ b/ci/conda/recipes/python-dbg/meta.yaml @@ -16,7 +16,7 @@ {% set version = environ.get('MORPHEUS_PYTHON_VER') %} package: - name: python + name: morpheus-python-dbg version: {{ version }} source: diff --git a/dependencies.yaml b/dependencies.yaml index 97ac66c53b..a31714a9bf 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -87,6 +87,8 @@ dependencies: - tqdm=4 - typing_utils=0.1 - watchdog=2.1 + - pip: + - pyarrow_hotfix test_python_morpheus: common: - output_types: [conda] From a66da916ad5dcdc2446baf34b140ca1244147300 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Tue, 12 Dec 2023 16:11:30 -0600 Subject: [PATCH 03/19] changes for local and github ci --- .../opt/morpheus/bin/post-attach-command.sh | 4 +- .github/workflows/ci_pipe.yml | 3 - .../workflows/{pull_request.yml => pr.yaml} | 11 ++ .pre-commit-config.yaml | 11 ++ ci/check_style.sh | 23 ++++ ci/release/update-version.sh | 4 +- ci/scripts/github/build.sh | 7 +- ci/scripts/github/checks.sh | 10 +- ci/scripts/github/common.sh | 32 +---- ci/scripts/github/docs.sh | 7 +- ci/scripts/github/test.sh | 7 +- ci/scripts/run_ci_local.sh | 7 +- dependencies.yaml | 112 +++++++++++++++++- 13 files changed, 191 insertions(+), 47 deletions(-) rename .github/workflows/{pull_request.yml => pr.yaml} (83%) create mode 100644 .pre-commit-config.yaml create mode 100755 ci/check_style.sh diff --git a/.devcontainer/opt/morpheus/bin/post-attach-command.sh b/.devcontainer/opt/morpheus/bin/post-attach-command.sh index c1384cba6d..43729d7367 100755 --- a/.devcontainer/opt/morpheus/bin/post-attach-command.sh +++ b/.devcontainer/opt/morpheus/bin/post-attach-command.sh @@ -27,6 +27,6 @@ ENV_NAME=${ENV_NAME:-morpheus} sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc; if conda_env_find "${ENV_NAME}" ; \ -then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml --prune; \ -else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml; \ +then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml --prune; \ +else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml; \ fi diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml index c1a92cdbd5..4989172fb3 100644 --- a/.github/workflows/ci_pipe.yml +++ b/.github/workflows/ci_pipe.yml @@ -132,7 +132,6 @@ jobs: env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} PARALLEL_LEVEL: '10' - MERGE_EXAMPLES_YAML: '1' strategy: fail-fast: true @@ -164,8 +163,6 @@ jobs: username: '$oauthtoken' password: ${{ secrets.NGC_API_KEY }} image: ${{ inputs.container }} - env: - MERGE_DOCS_YAML: '1' strategy: fail-fast: true diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pr.yaml similarity index 83% rename from .github/workflows/pull_request.yml rename to .github/workflows/pr.yaml index 52f513d6f5..c5390a28f2 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pr.yaml @@ -42,6 +42,17 @@ permissions: statuses: none jobs: + pr-builder: + needs: + - checks + - ci_pipe + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 + with: + enable_check_generated_files: false ci_pipe: uses: ./.github/workflows/ci_pipe.yml with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..030d61a3c0 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. + +repos: + - repo: https://github.com/rapidsai/dependency-file-generator + rev: v1.7.1 + hooks: + - id: rapids-dependency-file-generator + args: ["--clean"] + +default_language_version: + python: python3 \ No newline at end of file diff --git a/ci/check_style.sh b/ci/check_style.sh new file mode 100755 index 0000000000..82b6ab825d --- /dev/null +++ b/ci/check_style.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +rapids-logger "Create checks conda environment" +. /opt/conda/etc/profile.d/conda.sh + +rapids-dependency-file-generator \ + --output conda \ + --file_key checks \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n checks +conda activate checks + +# Run pre-commit checks +pre-commit run --all-files --show-diff-on-failure diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d072bbe2c9..dd8023344f 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -77,8 +77,8 @@ fi # Root CMakeLists.txt sed_runner 's/'"VERSION ${CURRENT_FULL_VERSION}.*"'/'"VERSION ${NEXT_FULL_VERSION}"'/g' CMakeLists.txt -# Conda environment file -sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" docker/conda/environments/cuda11.8_dev.yml +# Depedencies file +sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" dependencies.yaml # examples/digital_fingerprinting sed_runner "s/v${CURRENT_FULL_VERSION}-runtime/v${NEXT_FULL_VERSION}-runtime/g" \ diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh index c03cc60d84..35fcd2d9f8 100755 --- a/ci/scripts/github/build.sh +++ b/ci/scripts/github/build.sh @@ -18,7 +18,12 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh -update_conda_env +rapids-dependency-file-generator \ + --output conda \ + --file_key build \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +update_conda_env env.yaml log_toolchain diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh index 7426754590..d2b51f6405 100755 --- a/ci/scripts/github/checks.sh +++ b/ci/scripts/github/checks.sh @@ -18,7 +18,12 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh -update_conda_env +rapids-dependency-file-generator \ + --output conda \ + --file_key build \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +update_conda_env env.yaml log_toolchain @@ -55,9 +60,6 @@ export MORPHEUS_IN_SPHINX_BUILD=1 rapids-logger "Checking copyright headers" python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT} -rapids-logger "Running Python style checks" -${MORPHEUS_ROOT}/ci/scripts/python_checks.sh - rapids-logger "Checking versions" ${MORPHEUS_ROOT}/ci/scripts/version_checks.sh diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index 8756ea09bd..aebd474504 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -61,10 +61,6 @@ export SCCACHE_REGION="us-east-2" export SCCACHE_IDLE_TIMEOUT=32768 #export SCCACHE_LOG=debug -export CONDA_ENV_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_dev.yml -export CONDA_EXAMPLES_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_examples.yml -export CONDA_DOCS_YML=${MORPHEUS_ROOT}/docs/conda_docs.yml - export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_PYTHON_BUILD_STUBS=ON -DMORPHEUS_USE_CCACHE=ON" export FETCH_STATUS=0 @@ -75,31 +71,13 @@ function update_conda_env() { # Deactivate the environment first before updating conda deactivate - ENV_YAML=${CONDA_ENV_YML} - if [[ "${MERGE_EXAMPLES_YAML}" == "1" || "${MERGE_DOCS_YAML}" == "1" ]]; then - # Merge the dev, docs and examples envs, otherwise --prune will remove the examples packages - ENV_YAML=${WORKSPACE_TMP}/merged_env.yml - YAMLS="${CONDA_ENV_YML}" - if [[ "${MERGE_EXAMPLES_YAML}" == "1" ]]; then - YAMLS="${YAMLS} ${CONDA_EXAMPLES_YML}" - fi - if [[ "${MERGE_DOCS_YAML}" == "1" ]]; then - YAMLS="${YAMLS} ${CONDA_DOCS_YML}" - fi - - rapids-logger "Merging conda envs: ${YAMLS}" - conda run -n morpheus --live-stream conda-merge ${YAMLS} > ${ENV_YAML} - fi - - if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then - rapids-logger "Checking for updates to conda env" + rapids-logger "Checking for updates to conda env" - # Remove default/conflicting channels from base image - rm /opt/conda/.condarc + # Remove default/conflicting channels from base image + rm /opt/conda/.condarc - # Update the packages - rapids-mamba-retry env update -n morpheus --prune -q --file ${ENV_YAML} - fi + # Update the packages + rapids-mamba-retry env update -n morpheus --prune -q --file "$1" # Finally, reactivate conda activate morpheus diff --git a/ci/scripts/github/docs.sh b/ci/scripts/github/docs.sh index 4c2a5703f2..d03f79ebb0 100755 --- a/ci/scripts/github/docs.sh +++ b/ci/scripts/github/docs.sh @@ -18,7 +18,12 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh -update_conda_env +rapids-dependency-file-generator \ + --output conda \ + --file_key docs \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +update_conda_env env.yaml download_artifact "wheel.tar.bz" diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh index d96c2ed888..d93d778213 100755 --- a/ci/scripts/github/test.sh +++ b/ci/scripts/github/test.sh @@ -19,7 +19,12 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh /usr/bin/nvidia-smi -update_conda_env +rapids-dependency-file-generator \ + --output conda \ + --file_key test \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +update_conda_env env.yaml rapids-logger "Check versions" python3 --version diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh index 4b13ef4f7f..0a906aa45d 100755 --- a/ci/scripts/run_ci_local.sh +++ b/ci/scripts/run_ci_local.sh @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -exo pipefail + case "$1" in "" ) STAGES=("bash") @@ -74,14 +76,11 @@ for STAGE in "${STAGES[@]}"; do CONTAINER="${TEST_CONTAINER}" DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia --gpus all" if [[ "${STAGE}" == "test" ]]; then - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_EXAMPLES_YAML=1 --cap-add=sys_nice" + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --cap-add=sys_nice" fi else CONTAINER="${BUILD_CONTAINER}" DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc" - if [[ "${STAGE}" == "docs" ]]; then - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_DOCS_YAML=1" - fi fi if [[ "${STAGE}" == "bash" ]]; then diff --git a/dependencies.yaml b/dependencies.yaml index a31714a9bf..3a2d35f7cf 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -27,14 +27,62 @@ files: - test_python_morpheus - cudatoolkit + build: + output: none + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - build_cpp + - development_cpp + - run_morpheus + - test_python_morpheus + - cudatoolkit + + test: + output: none + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - build_cpp + - development_cpp + - run_morpheus + - test_python_morpheus + - cudatoolkit + + docs: + output: none + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - build_cpp + - development_cpp + - run_morpheus + - test_python_morpheus + - cudatoolkit + + runtime: + output: none + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - runtime + + channels: - conda-forge + - huggingface - rapidsai - nvidia/label/cuda-11.8.0 - nvidia - rapidsai-nightly - nvidia/label/dev - pytorch + - dglteam/label/cu118 + # - defaults dependencies: @@ -57,14 +105,66 @@ dependencies: - scikit-build=0.17.1 - tritonclient=2.26 # Required by NvTabular, force the version, so we get protobufs compatible with 4.21 - ucx=1.14 + development_cpp: common: - output_types: [conda] packages: - - clangdev=14 - - include-what-you-use=0.18 + - clangdev=16 + - include-what-you-use=0.20 - isort - yapf=0.40.1 + + docs: + common: + - output_types: [conda] + packages: + - breathe=4.34.0 + - doxygen=1.9.2 + - exhale=0.3.6 + - ipython + - myst-parser=0.17.2 + - nbsphinx + - sphinx + - sphinx_rtd_theme + + examples: + common: + - output_types: [conda] + packages: + - arxiv=1.4 + - boto3 + - cuml=23.06 + - dask>=2023.1.1 + - dgl=1.0.2 + - dill=0.3.6 + - distributed>=2023.1.1 + - huggingface_hub=0.10.1 # work-around for https://github.com/UKPLab/sentence-transformers/issues/1762 + - langchain=0.0.190 + - libwebp>=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863 + - mlflow>=2.2.1,<3 + - newspaper3k=0.2 + - openai=0.28 + - papermill=2.3.4 + - pypdf=3.16 + - requests-cache=1.1 + - s3fs>=2023.6 + - sentence-transformers + - transformers + ####### Pip Transitive Dependencies (keep sorted!) ####### + # These are dependencies that are available on conda, but are required by the pip packages listed below. Its much + # better to install them with conda than pip to allow for better dependency resolution. + - environs=9.5 + - minio=7.1 + - python-dotenv=1.0 + - ujson=5.8 + ####### Pip Dependencies (keep sorted!) ####### + - pip: + - google-search-results==2.4 + - grpcio-status==1.58 # To keep in sync with 1.58 grpcio which is installed for Morpheus + - nemollm + + run_morpheus: common: - output_types: [conda] @@ -89,6 +189,7 @@ dependencies: - watchdog=2.1 - pip: - pyarrow_hotfix + test_python_morpheus: common: - output_types: [conda] @@ -100,6 +201,13 @@ dependencies: - pip: - pytest-kafka==0.6.0 + runtime: + common: + - output_types: [conda] + packages: + - nb_conda_kernels + - pip + cudatoolkit: specific: - output_types: [conda] From 7f42af51666a3828930d17c42a5d550e697f0c86 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 13 Dec 2023 10:50:57 -0600 Subject: [PATCH 04/19] . --- .pre-commit-config.yaml | 17 +++++++++++++++-- ci/scripts/github/common.sh | 5 ++++- dependencies.yaml | 17 +++++++++++++---- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 030d61a3c0..956031a526 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,17 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. repos: - repo: https://github.com/rapidsai/dependency-file-generator @@ -8,4 +21,4 @@ repos: args: ["--clean"] default_language_version: - python: python3 \ No newline at end of file + python: python3 diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index aebd474504..d2643b2054 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -74,7 +74,9 @@ function update_conda_env() { rapids-logger "Checking for updates to conda env" # Remove default/conflicting channels from base image - rm /opt/conda/.condarc + if [[ -d /opt/conda/.condarc ]]; then + rm /opt/conda/.condarc + fi; # Update the packages rapids-mamba-retry env update -n morpheus --prune -q --file "$1" @@ -110,6 +112,7 @@ function fetch_base_branch_gh_api() { function fetch_base_branch_local() { rapids-logger "Retrieving base branch from git" + git remote remove upstream git remote add upstream ${GIT_UPSTREAM_URL} git fetch upstream --tags source ${MORPHEUS_ROOT}/ci/scripts/common.sh diff --git a/dependencies.yaml b/dependencies.yaml index 3a2d35f7cf..a7a991aca5 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -35,6 +35,7 @@ files: includes: - build_cpp - development_cpp + - benchmark_cpp - run_morpheus - test_python_morpheus - cudatoolkit @@ -58,10 +59,8 @@ files: arch: [x86_64] includes: - build_cpp - - development_cpp - - run_morpheus - - test_python_morpheus - cudatoolkit + - docs runtime: output: none @@ -90,12 +89,15 @@ dependencies: common: - output_types: [conda] packages: + - mrc=24.03 # should this be in build, or somewhere else? + - gcc_linux-64=11.2 + - gxx_linux-64=11.2 + - cmake=3.25 - boost-cpp=1.82 - cuda-nvcc - cudf=23.06 - cxx-compiler - cython=0.29.24 - - libhwloc=2.9.2 - librdkafka=1.9.2 - ninja=1.10 - nlohmann_json=3.9 @@ -119,6 +121,7 @@ dependencies: common: - output_types: [conda] packages: + - git-lfs - breathe=4.34.0 - doxygen=1.9.2 - exhale=0.3.6 @@ -128,6 +131,12 @@ dependencies: - sphinx - sphinx_rtd_theme + benchmark_cpp: + common: + - output_types: [conda] + packages: + - benchmark=1.6.1 + examples: common: - output_types: [conda] From 780a626316adecd485fdd9d6f674cedd33d6a66e Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Fri, 15 Dec 2023 19:25:19 +0000 Subject: [PATCH 05/19] update dependencies.yaml --- ci/scripts/run_ci_local.sh | 7 +++--- dependencies.yaml | 45 ++++++++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh index 0a906aa45d..96248fcc35 100755 --- a/ci/scripts/run_ci_local.sh +++ b/ci/scripts/run_ci_local.sh @@ -74,10 +74,9 @@ for STAGE in "${STAGES[@]}"; do DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}" if [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then CONTAINER="${TEST_CONTAINER}" - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia --gpus all" - if [[ "${STAGE}" == "test" ]]; then - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --cap-add=sys_nice" - fi + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia" + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --gpus all" + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --cap-add=sys_nice" else CONTAINER="${BUILD_CONTAINER}" DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc" diff --git a/dependencies.yaml b/dependencies.yaml index a7a991aca5..97e50d0ac5 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -23,7 +23,7 @@ files: includes: - build_cpp - development_cpp - - run_morpheus + - runtime - test_python_morpheus - cudatoolkit @@ -36,7 +36,7 @@ files: - build_cpp - development_cpp - benchmark_cpp - - run_morpheus + - runtime - test_python_morpheus - cudatoolkit @@ -46,10 +46,13 @@ files: cuda: ["11.8"] arch: [x86_64] includes: + - data_retrieval - build_cpp - development_cpp - - run_morpheus + - benchmark_cpp + - runtime - test_python_morpheus + - examples - cudatoolkit docs: @@ -58,9 +61,13 @@ files: cuda: ["11.8"] arch: [x86_64] includes: + - data_retrieval - build_cpp + - benchmark_cpp - cudatoolkit - docs + - runtime + - examples runtime: output: none @@ -108,6 +115,12 @@ dependencies: - tritonclient=2.26 # Required by NvTabular, force the version, so we get protobufs compatible with 4.21 - ucx=1.14 + data_retrieval: + common: + - output_types: [conda] + packages: + - git-lfs + development_cpp: common: - output_types: [conda] @@ -121,15 +134,18 @@ dependencies: common: - output_types: [conda] packages: - - git-lfs - breathe=4.34.0 - doxygen=1.9.2 - exhale=0.3.6 - ipython - myst-parser=0.17.2 - nbsphinx + - python-graphviz - sphinx - sphinx_rtd_theme + - numpydoc=1.4 + - pluggy=1.0 + - &click click >=8 benchmark_cpp: common: @@ -156,7 +172,6 @@ dependencies: - openai=0.28 - papermill=2.3.4 - pypdf=3.16 - - requests-cache=1.1 - s3fs>=2023.6 - sentence-transformers - transformers @@ -174,7 +189,7 @@ dependencies: - nemollm - run_morpheus: + runtime: common: - output_types: [conda] packages: @@ -182,41 +197,43 @@ dependencies: - click>=8 - datacompy=0.8 - dill + - elasticsearch==8.9.0 + - feedparser=6.0.10 - gputil - grpcio - mlflow>=2.2.1,<3 + - nb_conda_kernels - numpydoc=1.4 - nvtabular=23.06 - python-confluent-kafka=1.9.2 - python-graphviz - pytorch-cuda - pytorch=2.0.1 + - requests-cache=1.1 - scikit-learn=1.2.2 - sqlalchemy<2.0 # 2.0 is incompatible with pandas=1.3 - tqdm=4 - typing_utils=0.1 - watchdog=2.1 + - websockets - pip: + - databricks-connect + - milvus==2.3.2 - pyarrow_hotfix + - pymilvus==2.3.2 test_python_morpheus: common: - output_types: [conda] packages: - - pip + - nodejs=18.* - pytest + - pytest-asyncio - pytest-benchmark>=4.0 - pytest-cov - pip: - pytest-kafka==0.6.0 - runtime: - common: - - output_types: [conda] - packages: - - nb_conda_kernels - - pip - cudatoolkit: specific: - output_types: [conda] From a15ac9a03fd7b8548d6c7386893d246872378479 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Fri, 15 Dec 2023 19:32:01 +0000 Subject: [PATCH 06/19] add checks filekey to dependencies.yaml --- dependencies.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 97e50d0ac5..f06ba540db 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -40,6 +40,12 @@ files: - test_python_morpheus - cudatoolkit + checks: + common: + - output_types: [conda] + packages: + - pre-commit + test: output: none matrix: From 937e0c984f1d7f613dfe4c4ab74f28e12dec524f Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Fri, 15 Dec 2023 19:56:28 +0000 Subject: [PATCH 07/19] fix dependencies.yaml errors --- dependencies.yaml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index f06ba540db..57ed561cab 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -40,12 +40,6 @@ files: - test_python_morpheus - cudatoolkit - checks: - common: - - output_types: [conda] - packages: - - pre-commit - test: output: none matrix: @@ -83,6 +77,11 @@ files: includes: - runtime + checks: + output: none + includes: + - checks + channels: - conda-forge @@ -121,6 +120,12 @@ dependencies: - tritonclient=2.26 # Required by NvTabular, force the version, so we get protobufs compatible with 4.21 - ucx=1.14 + checks: + common: + - output_types: [conda] + packages: + - pre-commit + data_retrieval: common: - output_types: [conda] From 6c212473526ce11c23ef076a4ef6bc8a217b0aab Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Fri, 15 Dec 2023 20:25:58 +0000 Subject: [PATCH 08/19] fix .gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 4cb2d7fd83..a92915a355 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,7 +2,7 @@ path = external/morpheus-visualizations url = https://github.com/nv-morpheus/morpheus-visualizations.git branch = branch-24.03 -[submodule "morpheus_utils"] +[submodule "external/utilities"] path = external/utilities url = https://github.com/nv-morpheus/utilities.git branch = branch-24.03 From 416532dfee15ce1038e44268ab3dd52ad4613517 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Fri, 15 Dec 2023 23:02:58 +0000 Subject: [PATCH 09/19] . --- ci/scripts/github/test.sh | 4 +- .../all_cuda-118_arch-x86_64.yaml | 77 +++++++++++++++++++ dependencies.yaml | 3 + 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 conda/environments/all_cuda-118_arch-x86_64.yaml diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh index d93d778213..cb45189733 100755 --- a/ci/scripts/github/test.sh +++ b/ci/scripts/github/test.sh @@ -74,8 +74,8 @@ rapids-logger "Pulling LFS assets" git lfs install ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch tests validation -# List missing files -rapids-logger "Listing missing files" +# Listing LFS-known files +rapids-logger "Listing LFS-known files" git lfs ls-files REPORTS_DIR="${WORKSPACE_TMP}/reports" diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml new file mode 100644 index 0000000000..9f32a964aa --- /dev/null +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -0,0 +1,77 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +- huggingface +- rapidsai +- nvidia/label/cuda-11.8.0 +- nvidia +- rapidsai-nightly +- nvidia/label/dev +- pytorch +- dglteam/label/cu118 +dependencies: +- appdirs +- boost-cpp=1.82 +- clangdev=16 +- click>=8 +- cmake=3.25 +- cuda-cudart-dev=11.8 +- cuda-nvcc +- cuda-nvml-dev=11.8 +- cuda-nvrtc-dev=11.8 +- cuda-python=11.8.2 +- cuda-tools=11.8 +- cuda-version=11.8 +- cudf=23.06 +- cxx-compiler +- cython=0.29.24 +- datacompy=0.8 +- dill +- elasticsearch==8.9.0 +- feedparser=6.0.10 +- gcc_linux-64=11.2 +- gputil +- grpcio +- gxx_linux-64=11.2 +- include-what-you-use=0.20 +- isort +- librdkafka=1.9.2 +- mlflow>=2.2.1,<3 +- mrc=24.03 +- nb_conda_kernels +- ninja=1.10 +- nlohmann_json=3.9 +- nodejs=18.* +- numpydoc=1.4 +- nvtabular=23.06 +- pip +- protobuf=4.21.* +- pybind11-stubgen=0.10 +- pytest +- pytest-asyncio +- pytest-benchmark>=4.0 +- pytest-cov +- python-confluent-kafka=1.9.2 +- python-graphviz +- pytorch-cuda +- pytorch=2.0.1 +- rapidjson=1.1.0 +- requests-cache=1.1 +- scikit-build=0.17.1 +- scikit-learn=1.2.2 +- sqlalchemy<2.0 +- tqdm=4 +- tritonclient=2.26 +- typing_utils=0.1 +- ucx=1.14 +- watchdog=2.1 +- websockets +- yapf=0.40.1 +- pip: + - databricks-connect + - milvus==2.3.2 + - pyarrow_hotfix + - pymilvus==2.3.2 + - pytest-kafka==0.6.0 +name: all_cuda-118_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 57ed561cab..8d0d5a5a2d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -194,6 +194,7 @@ dependencies: - python-dotenv=1.0 - ujson=5.8 ####### Pip Dependencies (keep sorted!) ####### + - pip - pip: - google-search-results==2.4 - grpcio-status==1.58 # To keep in sync with 1.58 grpcio which is installed for Morpheus @@ -227,6 +228,7 @@ dependencies: - typing_utils=0.1 - watchdog=2.1 - websockets + - pip - pip: - databricks-connect - milvus==2.3.2 @@ -242,6 +244,7 @@ dependencies: - pytest-asyncio - pytest-benchmark>=4.0 - pytest-cov + - pip - pip: - pytest-kafka==0.6.0 From 87255b8a5b8f32f6738c2c99722b11d66212a08e Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Fri, 15 Dec 2023 23:06:30 +0000 Subject: [PATCH 10/19] . --- .../all_cuda-118_arch-x86_64.yaml | 37 +++++++++++++++++++ dependencies.yaml | 6 +++ 2 files changed, 43 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 9f32a964aa..2d7fdf540c 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -12,8 +12,13 @@ channels: - dglteam/label/cu118 dependencies: - appdirs +- arxiv=1.4 +- benchmark=1.6.1 - boost-cpp=1.82 +- boto3 +- breathe=4.34.0 - clangdev=16 +- click >=8 - click>=8 - cmake=3.25 - cuda-cudart-dev=11.8 @@ -24,53 +29,85 @@ dependencies: - cuda-tools=11.8 - cuda-version=11.8 - cudf=23.06 +- cuml=23.06 - cxx-compiler - cython=0.29.24 +- dask>=2023.1.1 - datacompy=0.8 +- dgl=1.0.2 - dill +- dill=0.3.6 +- distributed>=2023.1.1 +- doxygen=1.9.2 - elasticsearch==8.9.0 +- environs=9.5 +- exhale=0.3.6 - feedparser=6.0.10 - gcc_linux-64=11.2 +- git-lfs - gputil - grpcio - gxx_linux-64=11.2 +- huggingface_hub=0.10.1 - include-what-you-use=0.20 +- ipython - isort +- langchain=0.0.190 - librdkafka=1.9.2 +- libwebp>=1.3.2 +- minio=7.1 - mlflow>=2.2.1,<3 - mrc=24.03 +- myst-parser=0.17.2 - nb_conda_kernels +- nbsphinx +- newspaper3k=0.2 - ninja=1.10 - nlohmann_json=3.9 - nodejs=18.* - numpydoc=1.4 - nvtabular=23.06 +- openai=0.28 +- papermill=2.3.4 - pip +- pluggy=1.0 +- pre-commit - protobuf=4.21.* - pybind11-stubgen=0.10 +- pypdf=3.16 - pytest - pytest-asyncio - pytest-benchmark>=4.0 - pytest-cov - python-confluent-kafka=1.9.2 +- python-dotenv=1.0 - python-graphviz - pytorch-cuda - pytorch=2.0.1 - rapidjson=1.1.0 - requests-cache=1.1 +- s3fs>=2023.6 - scikit-build=0.17.1 - scikit-learn=1.2.2 +- sentence-transformers +- sphinx +- sphinx_rtd_theme - sqlalchemy<2.0 - tqdm=4 +- transformers - tritonclient=2.26 - typing_utils=0.1 - ucx=1.14 +- ujson=5.8 - watchdog=2.1 - websockets - yapf=0.40.1 - pip: - databricks-connect + - google-search-results==2.4 + - grpcio-status==1.58 - milvus==2.3.2 + - nemollm - pyarrow_hotfix - pymilvus==2.3.2 - pytest-kafka==0.6.0 diff --git a/dependencies.yaml b/dependencies.yaml index 8d0d5a5a2d..bdd2a430b8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -21,11 +21,17 @@ files: cuda: ["11.8"] arch: [x86_64] includes: + - data_retrieval - build_cpp - development_cpp + - benchmark_cpp + - examples - runtime - test_python_morpheus + - docs - cudatoolkit + - runtime + - checks build: output: none From d4b1ae93443b775afd0539b3c657bc83277b41bd Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 18 Dec 2023 18:44:21 +0000 Subject: [PATCH 11/19] downgrade benchmark to match mrc, pin transforms to be compatible with huggingface_hub --- dependencies.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index bdd2a430b8..a43c5edde8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -168,7 +168,7 @@ dependencies: common: - output_types: [conda] packages: - - benchmark=1.6.1 + - benchmark=1.6.0 examples: common: @@ -191,7 +191,7 @@ dependencies: - pypdf=3.16 - s3fs>=2023.6 - sentence-transformers - - transformers + - transformers=4.30.2 # newer versions are incompatible with our pinned version of huggingface_hub ####### Pip Transitive Dependencies (keep sorted!) ####### # These are dependencies that are available on conda, but are required by the pip packages listed below. Its much # better to install them with conda than pip to allow for better dependency resolution. From 4499da8f0baf6db536e008ca2ea5d0dc44a76c6e Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 18 Dec 2023 22:37:14 +0000 Subject: [PATCH 12/19] update environment files --- conda/environments/all_cuda-118_arch-x86_64.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 2d7fdf540c..3b37475027 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -13,7 +13,7 @@ channels: dependencies: - appdirs - arxiv=1.4 -- benchmark=1.6.1 +- benchmark=1.6.0 - boost-cpp=1.82 - boto3 - breathe=4.34.0 @@ -94,7 +94,7 @@ dependencies: - sphinx_rtd_theme - sqlalchemy<2.0 - tqdm=4 -- transformers +- transformers=4.30.2 - tritonclient=2.26 - typing_utils=0.1 - ucx=1.14 From 0978e5db0b27a57818fd0de7899d1513e62f7f90 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 18 Dec 2023 23:24:36 +0000 Subject: [PATCH 13/19] ignore generated environment files for copyright check --- ci/scripts/copyright.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/scripts/copyright.py b/ci/scripts/copyright.py index 3eab43e8c9..102afbb456 100755 --- a/ci/scripts/copyright.py +++ b/ci/scripts/copyright.py @@ -48,7 +48,8 @@ re.compile(r"^[^ \/\n]*build[^ \/\n]*\/.*$"), # Ignore any build*/ folder re.compile(r"^external\/.*$"), # Ignore external re.compile(r"[^ \/\n]*docs/source/(_lib|_modules|_templates)/.*$"), - re.compile(r"PULL_REQUEST_TEMPLATE.md"), # Ignore the PR template + re.compile(r"PULL_REQUEST_TEMPLATE.md"), # Ignore the PR template, + re.compile(r"[^ \/\n]*conda/environments/.*\.yaml$"), # Ignore generated environment files ] # this will break starting at year 10000, which is probably OK :) From 18a750015e73e2764a960e1ee813930dd92930bb Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 20 Dec 2023 18:56:39 +0000 Subject: [PATCH 14/19] add python checks back in --- ci/scripts/github/checks.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh index d2b51f6405..4abfacd157 100755 --- a/ci/scripts/github/checks.sh +++ b/ci/scripts/github/checks.sh @@ -60,6 +60,9 @@ export MORPHEUS_IN_SPHINX_BUILD=1 rapids-logger "Checking copyright headers" python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT} +rapids-logger "Running Python style checks" +${MORPHEUS_ROOT}/ci/scripts/python_checks.sh + rapids-logger "Checking versions" ${MORPHEUS_ROOT}/ci/scripts/version_checks.sh From 43713f05e49bd5599c9a75f0f21e286c2e5127b5 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 20 Dec 2023 19:04:10 +0000 Subject: [PATCH 15/19] add dev (no examples) environment file --- .../dev_cuda-118_arch-x86_64.yaml | 90 +++++++++++++++++++ dependencies.yaml | 17 ++++ 2 files changed, 107 insertions(+) create mode 100644 conda/environments/dev_cuda-118_arch-x86_64.yaml diff --git a/conda/environments/dev_cuda-118_arch-x86_64.yaml b/conda/environments/dev_cuda-118_arch-x86_64.yaml new file mode 100644 index 0000000000..3c75b0be50 --- /dev/null +++ b/conda/environments/dev_cuda-118_arch-x86_64.yaml @@ -0,0 +1,90 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +- huggingface +- rapidsai +- nvidia/label/cuda-11.8.0 +- nvidia +- rapidsai-nightly +- nvidia/label/dev +- pytorch +- dglteam/label/cu118 +dependencies: +- appdirs +- benchmark=1.6.0 +- boost-cpp=1.82 +- breathe=4.34.0 +- clangdev=16 +- click >=8 +- click>=8 +- cmake=3.25 +- cuda-cudart-dev=11.8 +- cuda-nvcc +- cuda-nvml-dev=11.8 +- cuda-nvrtc-dev=11.8 +- cuda-python=11.8.2 +- cuda-tools=11.8 +- cuda-version=11.8 +- cudf=23.06 +- cxx-compiler +- cython=0.29.24 +- datacompy=0.8 +- dill +- doxygen=1.9.2 +- elasticsearch==8.9.0 +- exhale=0.3.6 +- feedparser=6.0.10 +- gcc_linux-64=11.2 +- git-lfs +- gputil +- grpcio +- gxx_linux-64=11.2 +- include-what-you-use=0.20 +- ipython +- isort +- librdkafka=1.9.2 +- mlflow>=2.2.1,<3 +- mrc=24.03 +- myst-parser=0.17.2 +- nb_conda_kernels +- nbsphinx +- ninja=1.10 +- nlohmann_json=3.9 +- nodejs=18.* +- numpydoc=1.4 +- nvtabular=23.06 +- pip +- pluggy=1.0 +- pre-commit +- protobuf=4.21.* +- pybind11-stubgen=0.10 +- pytest +- pytest-asyncio +- pytest-benchmark>=4.0 +- pytest-cov +- python-confluent-kafka=1.9.2 +- python-graphviz +- pytorch-cuda +- pytorch=2.0.1 +- rapidjson=1.1.0 +- requests-cache=1.1 +- scikit-build=0.17.1 +- scikit-learn=1.2.2 +- sphinx +- sphinx_rtd_theme +- sqlalchemy<2.0 +- tqdm=4 +- tritonclient=2.26 +- typing_utils=0.1 +- ucx=1.14 +- watchdog=2.1 +- websockets +- yapf=0.40.1 +- pip: + - databricks-connect + - milvus==2.3.2 + - pyarrow_hotfix + - pymilvus==2.3.2 + - pytest-kafka==0.6.0 +name: dev_cuda-118_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index a43c5edde8..e0077eb020 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -33,6 +33,23 @@ files: - runtime - checks + dev: + output: conda + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - data_retrieval + - build_cpp + - development_cpp + - benchmark_cpp + - runtime + - test_python_morpheus + - docs + - cudatoolkit + - runtime + - checks + build: output: none matrix: From 5f3b3caac9f979535f6633ca1a0ef879ad1f7130 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 20 Dec 2023 20:19:56 +0000 Subject: [PATCH 16/19] add pylint to environment --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/dev_cuda-118_arch-x86_64.yaml | 1 + dependencies.yaml | 11 ++++++----- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 3b37475027..f151f4682b 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -74,6 +74,7 @@ dependencies: - pre-commit - protobuf=4.21.* - pybind11-stubgen=0.10 +- pylint>=2.17.4,<2.18 - pypdf=3.16 - pytest - pytest-asyncio diff --git a/conda/environments/dev_cuda-118_arch-x86_64.yaml b/conda/environments/dev_cuda-118_arch-x86_64.yaml index 3c75b0be50..cee8a9fe95 100644 --- a/conda/environments/dev_cuda-118_arch-x86_64.yaml +++ b/conda/environments/dev_cuda-118_arch-x86_64.yaml @@ -59,6 +59,7 @@ dependencies: - pre-commit - protobuf=4.21.* - pybind11-stubgen=0.10 +- pylint>=2.17.4,<2.18 - pytest - pytest-asyncio - pytest-benchmark>=4.0 diff --git a/dependencies.yaml b/dependencies.yaml index e0077eb020..0990f9d97d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -23,7 +23,7 @@ files: includes: - data_retrieval - build_cpp - - development_cpp + - development - benchmark_cpp - examples - runtime @@ -41,7 +41,7 @@ files: includes: - data_retrieval - build_cpp - - development_cpp + - development - benchmark_cpp - runtime - test_python_morpheus @@ -57,7 +57,7 @@ files: arch: [x86_64] includes: - build_cpp - - development_cpp + - development - benchmark_cpp - runtime - test_python_morpheus @@ -71,7 +71,7 @@ files: includes: - data_retrieval - build_cpp - - development_cpp + - development - benchmark_cpp - runtime - test_python_morpheus @@ -155,7 +155,7 @@ dependencies: packages: - git-lfs - development_cpp: + development: common: - output_types: [conda] packages: @@ -163,6 +163,7 @@ dependencies: - include-what-you-use=0.20 - isort - yapf=0.40.1 + - pylint>=2.17.4,<2.18 # 2.17.4 contains a fix for toml support docs: common: From c399bd521b132ffe608661fbae7e40282c4cc9cb Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 20 Dec 2023 20:22:19 +0000 Subject: [PATCH 17/19] add flake8 to dev environment --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/dev_cuda-118_arch-x86_64.yaml | 1 + dependencies.yaml | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index f151f4682b..d2557e605b 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -43,6 +43,7 @@ dependencies: - environs=9.5 - exhale=0.3.6 - feedparser=6.0.10 +- flake8 - gcc_linux-64=11.2 - git-lfs - gputil diff --git a/conda/environments/dev_cuda-118_arch-x86_64.yaml b/conda/environments/dev_cuda-118_arch-x86_64.yaml index cee8a9fe95..c0d127a07f 100644 --- a/conda/environments/dev_cuda-118_arch-x86_64.yaml +++ b/conda/environments/dev_cuda-118_arch-x86_64.yaml @@ -35,6 +35,7 @@ dependencies: - elasticsearch==8.9.0 - exhale=0.3.6 - feedparser=6.0.10 +- flake8 - gcc_linux-64=11.2 - git-lfs - gputil diff --git a/dependencies.yaml b/dependencies.yaml index 0990f9d97d..2fcc2f5b79 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -160,10 +160,11 @@ dependencies: - output_types: [conda] packages: - clangdev=16 + - flake8 - include-what-you-use=0.20 - isort - - yapf=0.40.1 - pylint>=2.17.4,<2.18 # 2.17.4 contains a fix for toml support + - yapf=0.40.1 docs: common: From c0e3fd279c0084177ca202b11137c314d1fffd2e Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 20 Dec 2023 21:20:32 +0000 Subject: [PATCH 18/19] keep .condarc in ci if it exists --- ci/scripts/github/common.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index d2643b2054..685ba7282a 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -73,11 +73,6 @@ function update_conda_env() { rapids-logger "Checking for updates to conda env" - # Remove default/conflicting channels from base image - if [[ -d /opt/conda/.condarc ]]; then - rm /opt/conda/.condarc - fi; - # Update the packages rapids-mamba-retry env update -n morpheus --prune -q --file "$1" From 42adf5a8fb7067c60d8c84e8c281602243ca56c0 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 20 Dec 2023 21:22:11 +0000 Subject: [PATCH 19/19] remove defaults comment in dependences.yaml --- dependencies.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 2fcc2f5b79..fd035b733d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -116,7 +116,6 @@ channels: - nvidia/label/dev - pytorch - dglteam/label/cu118 - # - defaults dependencies: