dependencies.yaml

# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Dependency list for https://github.com/rapidsai/dependency-file-generator

files:
  # Includes all dependencies together in a single file
  all:
    output: conda
    matrix:
      cuda: ["12.5"]
      arch: [x86_64]
    includes:
      - benchmark_cpp
      - build_cpp_build
      - build_cpp_host
      - checks
      - cudatoolkit
      - cudatoolkit-dev
      - cve-mitigation
      - data_retrieval
      - development
      - docs
      - example-dfp-prod
      - example-gnn
      - example-llms
      - python
      - runtime
      - test_python_morpheus

  # Common dependencies for developing with Morpheus.
  # Includes: runtime, build, docs, test
  # Excludes: examples
  dev:
    output: conda
    matrix:
      cuda: ["12.5"]
      arch: [x86_64]
    includes:
      - benchmark_cpp
      - build_cpp_build
      - build_cpp_host
      - checks
      - cudatoolkit-dev
      - cve-mitigation
      - data_retrieval
      - development
      - docs
      - python
      - runtime
      - test_python_morpheus

  # Dependencies that are needed to build Morpheus.
  # Includes: runtime
  # Excludes: test, examples
  build:
    output: none
    matrix:
      cuda: ["12.5"]
      arch: [x86_64]
    includes:
      - benchmark_cpp
      - build_cpp_build
      - build_cpp_host
      - cudatoolkit-dev
      - cve-mitigation
      - data_retrieval
      - development
      - python
      - runtime
      - test_python_morpheus

  # Dependencies that are needed to run Morpheus tests.
  # Includes: runtime, examples
  # Excludes: build, dev
  test:
    output: none
    matrix:
      cuda: ["12.5"]
      arch: [x86_64]
    includes:
      - benchmark_cpp
      - cudatoolkit
      - cve-mitigation
      - data_retrieval
      - development
      - example-dfp-prod
      - example-gnn
      - example-llms
      - python
      - runtime
      - test_python_morpheus

  # Dependencies that are needed to generate the Morpheus documentation.
  # Includes: build, runtime, examples
  # Excludes: test
  docs:
    output: none
    matrix:
      cuda: ["12.5"]
      arch: [x86_64]
    includes:
      - benchmark_cpp
      - build_cpp_build
      - build_cpp_host
      - cudatoolkit-dev
      - cve-mitigation
      - data_retrieval
      - docs
      - example-dfp-prod
      - example-gnn
      - example-llms
      - python
      - runtime

  # Only the dependencies that are needed to run core Morpheus. This is the smallest set of dependencies needed to run
  # a Morpheus pipeline.
  # Includes: none
  # Excludes: build, dev, test, examples, docs
  runtime:
    output: conda
    matrix:
      cuda: ["12.5"]
      arch: [x86_64]
    includes:
      - cudatoolkit
      - cve-mitigation
      - python
      - runtime

  # Dependencies that are needed to run all Morpheus examples.
  # Includes: runtime
  # Excludes: build, dev, test
  examples:
    output: conda
    matrix:
      cuda: ["12.5"]
      arch: [x86_64]
    includes:
      - cve-mitigation
      - example-abp-nvsmi
      - example-dfp-prod
      - example-gnn
      - example-llms
      - python
      - runtime

  # Dependencies which are needed to run the model generation scripts in the `models` directory.
  # Includes: none
  # Excludes: runtime, build, dev, test, examples, docs
  model-utils:
    output: conda
    matrix:
      cuda: ["12.5"]
      arch: [x86_64]
    includes:
      - model-training-tuning
      - python

  # Dependencies which are used during the CI "checks" stage. Does not output any files.
  checks:
    output: none
    includes:
      - checks
      - python

  # morpheus-core pip dependencies are published as a morpheus package data file
  morpheus_core:
    output: requirements
    requirements_dir: python/morpheus/morpheus
    includes:
      - morpheus_core_pip

  # morpheus-dfp pip dependencies are published as a morpheus_dfp package data file
  morpheus_dfp:
    output: requirements
    requirements_dir: python/morpheus_dfp/morpheus_dfp
    includes:
      - morpheus_core_pip

  # morpheus-llm pip dependencies are published as a morpheus_llm package data file
  morpheus_llm:
    output: requirements
    requirements_dir: python/morpheus_llm/morpheus_llm
    includes:
      - morpheus_core_pip
      - morpheus_llm_pip

  # dfp example conda dependencies
  dfp_example:
    output: conda
    matrix:
      cuda: ["12.5"]
      arch: [x86_64]
    output: conda
    conda_dir: examples/digital_fingerprinting/production/conda/environments
    includes:
      - example-dfp-container
      - example-dfp-prod

channels:
  - conda-forge
  - huggingface
  - rapidsai
  - rapidsai-nightly
  - nvidia
  - nvidia/label/dev
  - pytorch

dependencies:

  python:
    common:
      - output_types: [conda]
        packages:
          - python=3.10

  cudatoolkit:
    specific:
      - output_types: [conda]
        matrices:
          - matrix:
              cuda: "12.5"
            packages:
              - cuda-cudart=12.5
              - cuda-nvrtc=12.5
              - cuda-nvtx=12.5
              - cuda-version=12.5

  cudatoolkit-dev:
    specific:
      - output_types: [conda]
        matrices:
          - matrix:
              cuda: "12.5"
            packages:
              - cuda-cudart-dev=12.5
              - cuda-nvml-dev=12.5
              - cuda-nvrtc-dev=12.5
              - cuda-nvtx-dev=12.5
              - cuda-sanitizer-api
              - cuda-version=12.5
              - libcublas-dev # required by matx
              - libcufft-dev # required by matx
              - libcusolver-dev # required by matx
              - libcurand-dev # required by matx



  # Build dependencies for Morpheus on the target arch. Mirrors the `build` section in
  # ci/conda/recipes/morpheus/meta.yaml
  build_cpp_build:
    common:
      - output_types: [conda]
        packages:
          # Compilers
          - cuda-nvcc=12.5
          - cxx-compiler
          - gxx=12.1

          # Non-Compiler Dependencies
          - automake=1.16.5 # Needed for DOCA build
          - c-ares=1.32 # 1.33 causes an undefined symbol error
          - ccache
          - cmake=3.27
          - cuda-cudart-dev=12.5
          - cuda-version=12.5
          - indicators=2.3 # C++ library for displaying progress bars
          - libtool # Needed for DOCA build
          - ninja=1.11
          - pkg-config=0.29 # for mrc cmake
          - sysroot_linux-64>=2.28

  # Build dependencies for Morpheus on the host arch. Mirrors the `host` section in
  # ci/conda/recipes/morpheus/meta.yaml
  build_cpp_host:
    common:
      - output_types: [conda]
        packages:
          # Include: cudatoolkit-dev
          - cudf=24.10
          - cython=3.0
          - glog>=0.7.1,<0.8
          - gtest=1.14
          - libcudf=24.10
          - librdkafka>=1.9.2,<1.10.0a0
          - libzlib >=1.3.1,<2
          - mrc=25.02
          - nlohmann_json=3.11
          - pybind11-stubgen=0.10.5
          - pylibcudf=24.10
          - rapidjson=1.1.0
          - rdma-core>=48 # Needed for DOCA.
          - scikit-build=0.17.6
          - versioneer-518

  checks:
    common:
      - output_types: [conda]
        packages:
          - pre-commit

  data_retrieval:
    common:
      - output_types: [conda]
        packages:
          - git-lfs

  development:
    common:
      - output_types: [conda]
        packages:
          - clangdev=16
          - flake8
          - include-what-you-use=0.20
          - isort
          - pylint=3.0.3
          - pynvml=11.4
          - vale=3.7
          - vale-styles-microsoft
          - vale-styles-write-good
          - versioneer
          - yapf=0.40.1

  docs:
    common:
      - output_types: [conda]
        packages:
          - breathe=4.35.0
          - doxygen=1.9.2
          - exhale=0.3.6
          - ipython
          - myst-parser=0.18.1
          - nbsphinx
          - sphinx
          - sphinx_rtd_theme

  benchmark_cpp:
    common:
      - output_types: [conda]
        packages:
          - benchmark=1.8.3

  # Runtime dependencies for Morpheus. Included in nearly all output files so dependencies should
  # be added only if it is needed to run the core Morpheus library.
  # This should be synced with `run` in ci/conda/recipes/morpheus/meta.yaml
  runtime:
    common:
      - output_types: [conda]
        packages:
          # Include: cudatoolkit
          # Include: python
          # Include: cve-mitigation
          - appdirs
          - beautifulsoup4=4.12
          - click>=8
          # - cuda-version=12.5 ##
          - cudf=24.10
          - cupy # Version determined from cudf
          - datacompy=0.10
          - dill=0.3.7
          - docker-py=5.0
          - elasticsearch==8.9.0
          - feedparser=6.0
          - grpcio
          - grpcio-status
          # - libwebp=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863 ##
          - mlflow>=2.10.0,<2.18 # Pin version to avoid breaking change in 2.18 to thread local variable code commit id: 5541888
          - mrc=25.02
          - networkx=2.8.8
          - numpydoc=1.5
          - pydantic
          - python-confluent-kafka>=1.9.2,<1.10.0a0
          - python-graphviz
          - pluggy=1.3
          - rapids-dask-dependency=24.10 # provides dask and distributed
          - requests
          - requests-cache=1.1
          - scikit-learn=1.3.2
          - sqlalchemy<2.0 # 2.0 is incompatible with pandas=1.3
          - tqdm=4
          - tritonclient=2.34
          - typing_utils=0.1
          - watchdog=3.0
          - websockets
          - pip
          - pip:
            - --extra-index-url https://download.pytorch.org/whl/cu124
            - databricks-cli < 0.100
            - databricks-connect
            - milvus==2.3.5 # update to match pymilvus when available
            - pymilvus==2.3.6
            - torch==2.4.0+cu124

  test_python_morpheus:
    common:
      - output_types: [conda]
        packages:
          - &nodejs nodejs=18.*
          - &pypdfium2 pypdfium2=4.30
          - pytest-asyncio
          - pytest-benchmark=4.0
          - pytest-cov
          - pytest=7.4.4
          - &python-docx python-docx==1.1.0
          - pip
          - pip:
              - pytest-kafka==0.6.0

  morpheus_core_pip:
    common:
      - output_types: [requirements]
        packages:
           - --extra-index-url https://download.pytorch.org/whl/cu124
           - torch==2.4.0+cu124

  morpheus_llm_pip:
    common:
      - output_types: [requirements]
        packages:
           - &faiss-cpu faiss-cpu
           - &google-search-results google-search-results==2.4
           - &langchain langchain==0.1.16
           - &langchain-nvidia-ai-endpoints langchain-nvidia-ai-endpoints==0.0.11
           - &langchain-openai langchain-openai==0.1.3
           - milvus==2.3.5 # update to match pymilvus when available
           - &openai openai==1.13.*
           - pymilvus==2.3.6
           - &nemollm nemollm==0.3.5

  example-dfp-prod:
    common:
      - output_types: [conda]
        packages:
          - *nodejs
          - boto3=1.35
          - kfp
          - papermill=2.4.0
          - s3fs=2024.10
          - pip
          - pip:
            - python-logging-loki

  # packages needed in the DFP example container
  example-dfp-container:
    common:
      - output_types: [conda]
        packages:
          - morpheus-dfp=24.10
          - tini=0.19
          - pip:
            - --extra-index-url https://download.pytorch.org/whl/cu124
            - torch==2.4.0+cu124

  example-gnn:
    common:
      - output_types: [conda]
        packages:
          - &cuml cuml=24.10.*
          - pip
          - pip:
            - --find-links https://data.dgl.ai/wheels/cu121/repo.html
            - --find-links https://data.dgl.ai/wheels-test/repo.html
            - dgl==2.0.0
            - dglgo

  example-abp-nvsmi:
    common:
      - output_types: [conda]
        packages:
          - pynvml=11.4

  example-llms:
    common:
      - output_types: [conda]
        packages:
          - &transformers transformers=4.36.2 # newer versions are incompatible with our pinned version of huggingface_hub
          - anyio>=3.7
          - arxiv=1.4
          - httpx>=0.23,<0.28 # work-around for https://github.com/openai/openai-python/issues/1915
          - huggingface_hub=0.20.2 # work-around for https://github.com/UKPLab/sentence-transformers/issues/1762
          - jsonpatch>=1.33
          - newspaper3k=0.2
          - numexpr
          - onnx=1.15
          - *openai
          - pypdf=3.17.4
          - *pypdfium2
          - *python-docx
          - requests-toolbelt=1.0 # Transitive dep needed by nemollm, specified here to ensure we get a compatible version
          - pip
          - pip:
            - *faiss-cpu
            - *google-search-results
            - *langchain
            - *langchain-nvidia-ai-endpoints
            - *langchain-openai
            - *nemollm
            - sentence-transformers==2.7 # using pip now instead of conda to avoid install of pytorch cpu

  model-training-tuning:
    common:
      - output_types: [conda]
        packages:
          - *cuml
          - *transformers
          - scikit-learn=1.3.2
          - jupyterlab
          - matplotlib
          - onnx
          - pandas
          - seaborn
          - seqeval=1.2.2
          - xgboost
          - pip
          - pip:
            - tensorrt-cu12

  cve-mitigation:
    common:
      - output_types: [conda]
        packages:
          - libwebp=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863