Updates related to the rapids-24.10 upgrade

Signed-off-by: Anuradha Karuppiah <[email protected]>

ci/conda/recipes/morpheus-libs/meta.yaml

@@ -121,6 +121,7 @@ outputs:
         - pytest-cov
         - pytest =7.4.4
       source_files:
+        - dependencies.yaml
         - pyproject.toml
         - scripts/fetch_data.py
         - tests/*
@@ -174,6 +175,7 @@ outputs:
         - pytest-cov
         - pytest =7.4.4
       source_files:
+        - dependencies.yaml
         - pyproject.toml
         - scripts/fetch_data.py
         - tests/*

ci/conda/recipes/morpheus-libs/morpheus_core_test.sh

@@ -13,4 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# morpheus tests are dependent on some pypi packages
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key morpheus_core --matrix "" | tee "core_requirements.txt"
+
+pip install -r core_requirements.txt
+
 pytest tests/morpheus

ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh

@@ -13,4 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# morpheus tests are dependent on some pypi packages
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key morpheus_dfp --matrix "" | tee "dfp_requirements.txt"
+
+pip install -r dfp_requirements.txt
+
 pytest tests/morpheus_dfp

ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh

@@ -16,7 +16,7 @@
 # llm library and tests are dependent on a number of pypi packages - fixme
 rapids-dependency-file-generator \
   --output requirements \
-  --file_key morpheus_llm --matrix "" | tee "llm_requirements.txt"
+  --file-key morpheus_llm --matrix "" | tee "llm_requirements.txt"
 
 pip install -r llm_requirements.txt
 

ci/scripts/run_ci_local.sh

@@ -59,11 +59,12 @@ GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp}
 CONTAINER_VER=${CONTAINER_VER:-241004}
-CUDA_VER=${CUDA_VER:-12.1}
+CUDA_VER=${CUDA_VER:-12.5}
+CUDA_FULL_VER=12.5.1
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 
 # Configure the base docker img
-CONDA_CONTAINER="rapidsai/ci-conda:cuda12.1.1-ubuntu22.04-py3.10"
+CONDA_CONTAINER="rapidsai/ci-conda:cuda${CUDA_FULL_VER}-ubuntu22.04-py3.10"
 BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-${CONTAINER_VER}"
 TEST_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-${CONTAINER_VER}"
 

dependencies.yaml

@@ -181,6 +181,18 @@ files:
     includes:
       - test_morpheus_llm_pip
 
+  # pip dependencies that are used during the conda test stage for morpheus-core output
+  morpheus_core:
+    output: none
+    includes:
+      - test_morpheus_core_pip
+
+  # pip dependencies that are used during the conda test stage for morpheus-dfp output
+  morpheus_dfp:
+    output: none
+    includes:
+      - test_morpheus_core_pip
+
 channels:
   - conda-forge
   - huggingface
@@ -388,6 +400,13 @@ dependencies:
           - pip:
               - pytest-kafka==0.6.0
 
+  test_morpheus_core_pip:
+    common:
+      - output_types: [requirements]
+        packages:
+           - --extra-index-url https://download.pytorch.org/whl/cu124
+           - torch==2.4.0+cu124
+
   test_morpheus_llm_pip:
     common:
       - output_types: [requirements]
@@ -396,7 +415,7 @@ dependencies:
            - pymilvus==2.3.6
            - langchain==0.1.16
            - langchain-nvidia-ai-endpoints==0.0.11
-           - faiss-gpu==1.7.*
+           - faiss-cpu
            - google-search-results==2.4
             - nemollm==0.3.5