From b2311dc1b989c8387a0d5ef38efdd1df0c4ec16f Mon Sep 17 00:00:00 2001 From: Anuradha Karuppiah Date: Thu, 10 Oct 2024 18:01:25 +0000 Subject: [PATCH] Updates related to the rapids-24.10 upgrade Signed-off-by: Anuradha Karuppiah --- ci/conda/recipes/morpheus-libs/meta.yaml | 2 ++ .../morpheus-libs/morpheus_core_test.sh | 7 +++++++ .../morpheus-libs/morpheus_dfp_test.sh | 7 +++++++ .../morpheus-libs/morpheus_llm_test.sh | 2 +- ci/scripts/run_ci_local.sh | 5 +++-- dependencies.yaml | 21 ++++++++++++++++++- 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ci/conda/recipes/morpheus-libs/meta.yaml b/ci/conda/recipes/morpheus-libs/meta.yaml index 9d6ad5d029..747f734371 100644 --- a/ci/conda/recipes/morpheus-libs/meta.yaml +++ b/ci/conda/recipes/morpheus-libs/meta.yaml @@ -121,6 +121,7 @@ outputs: - pytest-cov - pytest =7.4.4 source_files: + - dependencies.yaml - pyproject.toml - scripts/fetch_data.py - tests/* @@ -174,6 +175,7 @@ outputs: - pytest-cov - pytest =7.4.4 source_files: + - dependencies.yaml - pyproject.toml - scripts/fetch_data.py - tests/* diff --git a/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh index 44e5d7805b..d7379cda4c 100644 --- a/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh +++ b/ci/conda/recipes/morpheus-libs/morpheus_core_test.sh @@ -13,4 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +# morpheus tests are dependent on some pypi packages +rapids-dependency-file-generator \ + --output requirements \ + --file-key morpheus_core --matrix "" | tee "core_requirements.txt" + +pip install -r core_requirements.txt + pytest tests/morpheus diff --git a/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh index d6a749d770..2c38db4cb3 100644 --- a/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh +++ b/ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh @@ -13,4 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +# morpheus tests are dependent on some pypi packages +rapids-dependency-file-generator \ + --output requirements \ + --file-key morpheus_dfp --matrix "" | tee "dfp_requirements.txt" + +pip install -r dfp_requirements.txt + pytest tests/morpheus_dfp diff --git a/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh b/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh index 13b4b0b5bf..67e7b2f52c 100644 --- a/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh +++ b/ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh @@ -16,7 +16,7 @@ # llm library and tests are dependent on a number of pypi packages - fixme rapids-dependency-file-generator \ --output requirements \ - --file_key morpheus_llm --matrix "" | tee "llm_requirements.txt" + --file-key morpheus_llm --matrix "" | tee "llm_requirements.txt" pip install -r llm_requirements.txt diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh index 5a7971b340..86749de020 100755 --- a/ci/scripts/run_ci_local.sh +++ b/ci/scripts/run_ci_local.sh @@ -59,11 +59,12 @@ GIT_COMMIT=$(git log -n 1 --pretty=format:%H) LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp} CONTAINER_VER=${CONTAINER_VER:-241004} -CUDA_VER=${CUDA_VER:-12.1} +CUDA_VER=${CUDA_VER:-12.5} +CUDA_FULL_VER=12.5.1 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} # Configure the base docker img -CONDA_CONTAINER="rapidsai/ci-conda:cuda12.1.1-ubuntu22.04-py3.10" +CONDA_CONTAINER="rapidsai/ci-conda:cuda${CUDA_FULL_VER}-ubuntu22.04-py3.10" BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-${CONTAINER_VER}" TEST_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-${CONTAINER_VER}" diff --git a/dependencies.yaml b/dependencies.yaml index f14de2c44e..7b7002e89d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -181,6 +181,18 @@ files: includes: - test_morpheus_llm_pip + # pip dependencies that are used during the conda test stage for morpheus-core output + morpheus_core: + output: none + includes: + - test_morpheus_core_pip + + # pip dependencies that are used during the conda test stage for morpheus-dfp output + morpheus_dfp: + output: none + includes: + - test_morpheus_core_pip + channels: - conda-forge - huggingface @@ -388,6 +400,13 @@ dependencies: - pip: - pytest-kafka==0.6.0 + test_morpheus_core_pip: + common: + - output_types: [requirements] + packages: + - --extra-index-url https://download.pytorch.org/whl/cu124 + - torch==2.4.0+cu124 + test_morpheus_llm_pip: common: - output_types: [requirements] @@ -396,7 +415,7 @@ dependencies: - pymilvus==2.3.6 - langchain==0.1.16 - langchain-nvidia-ai-endpoints==0.0.11 - - faiss-gpu==1.7.* + - faiss-cpu - google-search-results==2.4 - nemollm==0.3.5