Include requirements files in the morpheus packages (#1957)

pip dependencies are included as a part of the package -
morpheus/requirements_morpheus_core.txt
morpheus_dfp/requirements_morpheus_dfp.txt
morpheus_llm/requirements_morpheus_llm.txt

And can be installed as needed. For example -
```
import importlib.resources
import subprocess
requirements_file = importlib.resources.path("morpheus_llm", "requirements_morpheus_llm.txt")
subprocess.call(f"pip install -r {requirements_file}".split())
```

Authors:
  - Anuradha Karuppiah (https://github.com/AnuradhaKaruppiah)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #1957

ci/conda/recipes/morpheus-libs/morpheus_core_test.sh

@@ -13,11 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# morpheus tests are dependent on some pypi packages
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_core --matrix "" | tee "core_requirements.txt"
-
-pip install -r core_requirements.txt
+python3 <<EOF
+import importlib.resources
+import subprocess
+requirements_file = importlib.resources.path("morpheus", "requirements_morpheus_core.txt")
+subprocess.call(f"pip install -r {requirements_file}".split())
+EOF
 
 pytest tests/morpheus

ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh

@@ -13,11 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# morpheus tests are dependent on some pypi packages
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_dfp --matrix "" | tee "dfp_requirements.txt"
-
-pip install -r dfp_requirements.txt
+python3 <<EOF
+import importlib.resources
+import subprocess
+requirements_file = importlib.resources.path("morpheus_dfp", "requirements_morpheus_dfp.txt")
+subprocess.call(f"pip install -r {requirements_file}".split())
+EOF
 
 pytest tests/morpheus_dfp

ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh

@@ -13,11 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# llm library and tests are dependent on a number of pypi packages - fixme
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_llm --matrix "" | tee "llm_requirements.txt"
-
-pip install -r llm_requirements.txt
+# Install requirements if they are included in the package
+python3 <<EOF
+import importlib.resources
+import subprocess
+requirements_file = importlib.resources.path("morpheus_llm", "requirements_morpheus_llm.txt")
+subprocess.call(f"pip install -r {requirements_file}".split())
+EOF
 
 pytest tests/morpheus_llm

dependencies.yaml

@@ -176,24 +176,27 @@ files:
       - checks
       - python
 
-  # pip dependencies that are used during the conda test stage for morpheus-llm output
-  morpheus_llm:
-    output: none
-    includes:
-      - test_morpheus_llm_pip
-      - test_morpheus_core_pip
-
-  # pip dependencies that are used during the conda test stage for morpheus-core output
+  # morpheus-core pip dependencies are published as a morpheus package data file
   morpheus_core:
-    output: none
+    output: requirements
+    requirements_dir: python/morpheus/morpheus
     includes:
-      - test_morpheus_core_pip
+      - morpheus_core_pip
 
-  # pip dependencies that are used during the conda test stage for morpheus-dfp output
+  # morpheus-dfp pip dependencies are published as a morpheus_dfp package data file
   morpheus_dfp:
-    output: none
+    output: requirements
+    requirements_dir: python/morpheus_dfp/morpheus_dfp
+    includes:
+      - morpheus_core_pip
+
+  # morpheus-llm pip dependencies are published as a morpheus_llm package data file
+  morpheus_llm:
+    output: requirements
+    requirements_dir: python/morpheus_llm/morpheus_llm
     includes:
-      - test_morpheus_core_pip
+      - morpheus_core_pip
+      - morpheus_llm_pip
 
 channels:
   - conda-forge
@@ -402,24 +405,24 @@ dependencies:
           - pip:
               - pytest-kafka==0.6.0
 
-  test_morpheus_core_pip:
+  morpheus_core_pip:
     common:
       - output_types: [requirements]
         packages:
            - --extra-index-url https://download.pytorch.org/whl/cu124
            - torch==2.4.0+cu124
 
-  test_morpheus_llm_pip:
+  morpheus_llm_pip:
     common:
       - output_types: [requirements]
         packages:
-           - milvus==2.3.5 # update to match pymilvus when available
-           - pymilvus==2.3.6
-           - langchain==0.1.16
-           - langchain-nvidia-ai-endpoints==0.0.11
            - faiss-cpu
            - google-search-results==2.4
-            - nemollm==0.3.5
+           - langchain==0.1.16
+           - langchain-nvidia-ai-endpoints==0.0.11
+           - milvus==2.3.5 # update to match pymilvus when available
+           - pymilvus==2.3.6
+           - nemollm==0.3.5
 
   example-dfp-prod:
     common:

python/morpheus/CMakeLists.txt

@@ -50,6 +50,9 @@ endif()
 file(GLOB morpheus_data_files "${CMAKE_CURRENT_SOURCE_DIR}/morpheus/data/*")
 morpheus_utils_add_python_sources(${morpheus_data_files})
 
+file(GLOB morpheus_requirement_files "${CMAKE_CURRENT_SOURCE_DIR}/morpheus/requirements*")
+morpheus_utils_add_python_sources(${morpheus_requirement_files})
+
 morpheus_utils_build_python_package(morpheus ${extra_args})
 
 list(POP_BACK CMAKE_MESSAGE_CONTEXT)

python/morpheus/MANIFEST.in

@@ -1,3 +1,4 @@
 include morpheus/_version.py
+include morpheus/requirements_morpheus_core.txt
 recursive-include morpheus/data *
 recursive-include morpheus *.cpython*.so py.typed *.pyi

python/morpheus/morpheus/requirements_morpheus_core.txt

@@ -0,0 +1,4 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+torch==2.4.0+cu124

python/morpheus_dfp/CMakeLists.txt

@@ -41,6 +41,9 @@ if(MORPHEUS_PYTHON_PERFORM_INSTALL)
   list(APPEND extra_args "INSTALL_WHEEL")
 endif()
 
+file(GLOB morpheus_dfp_requirement_files "${CMAKE_CURRENT_SOURCE_DIR}/morpheus_dfp/requirements*")
+morpheus_utils_add_python_sources(${morpheus_dfp_requirement_files})
+
 morpheus_utils_build_python_package(morpheus_dfp ${extra_args})
 
 list(POP_BACK CMAKE_MESSAGE_CONTEXT)

python/morpheus_dfp/MANIFEST.in

@@ -1,2 +1,3 @@
 include morpheus_dfp/_version.py
+include morpheus_dfp/requirements_morpheus_dfp.txt
 recursive-include morpheus_dfp py.typed

python/morpheus_dfp/morpheus_dfp/requirements_morpheus_dfp.txt

@@ -0,0 +1,4 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+torch==2.4.0+cu124

python/morpheus_llm/CMakeLists.txt

@@ -45,6 +45,9 @@ if(MORPHEUS_PYTHON_PERFORM_INSTALL)
   list(APPEND extra_args "INSTALL_WHEEL")
 endif()
 
+file(GLOB morpheus_llm_requirement_files "${CMAKE_CURRENT_SOURCE_DIR}/morpheus_llm/requirements*")
+morpheus_utils_add_python_sources(${morpheus_llm_requirement_files})
+
 morpheus_utils_build_python_package(morpheus_llm ${extra_args})
 
 list(POP_BACK CMAKE_MESSAGE_CONTEXT)

python/morpheus_llm/MANIFEST.in

@@ -1,2 +1,3 @@
 include morpheus_llm/_version.py
+include morpheus_llm/requirements_morpheus_llm.txt
 recursive-include morpheus_llm *.cpython*.so py.typed *.pyi

python/morpheus_llm/morpheus_llm/requirements_morpheus_llm.txt

@@ -0,0 +1,11 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+faiss-cpu
+google-search-results==2.4
+langchain-nvidia-ai-endpoints==0.0.11
+langchain==0.1.16
+milvus==2.3.5
+nemollm==0.3.5
+pymilvus==2.3.6
+torch==2.4.0+cu124