diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 9195d475d4..86a42253fb 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +13,92 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM rapidsai/devcontainers:23.12-cpp-cuda12.1-mambaforge-ubuntu22.04 AS base +ARG BASE +ARG PYTHON_PACKAGE_MANAGER -ENV PATH="${PATH}:/workspaces/morpheus/.devcontainer/bin" +FROM node:22 as node + +FROM ${BASE} as base + +# ===== install common packages ================================================================== + +RUN </dev/null' + +ln -s /usr/local/bin/node /usr/local/bin/nodejs +ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm +ln -s /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx + +echo "node version: $(node --version)" +echo " npm version: $(npm --version)" +echo "yarn version: $(yarn --version)" + +npm install -g camouflage-server@0.15 + +EOF + + +FROM base as pip-base + +ENV DEFAULT_VIRTUAL_ENV=morpheus + +FROM base as conda-base + +ENV DEFAULT_CONDA_ENV=morpheus + +FROM ${PYTHON_PACKAGE_MANAGER}-base + +ENV PROJECT_MANIFEST_YML="/home/coder/morpheus/manifest.yaml" +ENV PATH="${PATH}:/home/coder/morpheus/.devcontainer/bin" + +ARG CUDA +ENV CUDAARCHS="RAPIDS" +ENV CUDA_VERSION="${CUDA_VERSION:-${CUDA}}" + +ARG RAPIDS +ENV RAPIDS=${RAPIDS} + +ARG PYTHON_PACKAGE_MANAGER +ENV PYTHON_PACKAGE_MANAGER="${PYTHON_PACKAGE_MANAGER}" + +ENV SCCACHE_REGION="us-east-2" +ENV SCCACHE_BUCKET="rapids-sccache-devs" +ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai" +ENV HISTFILE="/home/coder/.cache/._bash_history" + +ENV MORPHEUS_SUPPORT_DOCA=ON +COPY ./docker/optional_deps docker/optional_deps +RUN ./docker/optional_deps/doca.sh /tmp/doca diff --git a/.devcontainer/cuda12.1-conda/devcontainer.json b/.devcontainer/cuda12.1-conda/devcontainer.json new file mode 100644 index 0000000000..fa5d24c6d8 --- /dev/null +++ b/.devcontainer/cuda12.1-conda/devcontainer.json @@ -0,0 +1,112 @@ +{ + "build": { + "context": "${localWorkspaceFolder}/.devcontainer", + "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", + "args": { + "CUDA": "12.1", + "PYTHON_PACKAGE_MANAGER": "conda", + "BASE": "rapidsai/devcontainers:24.06-cpp-mambaforge-ubuntu22.04" + } + }, + "privileged": true, + "hostRequirements": { + "gpu": "optional" + }, + "capAdd": [ + "SYS_NICE", + "SYS_PTRACE" + ], + "securityOpt": [ + "seccomp=unconfined" + ], + "runArgs": [ + "--network=morpheus" + ], + "containerEnv": { + "HOST_MORPHEUS_ROOT": "${localWorkspaceFolder}", + "MORPHEUS_ROOT": "${containerWorkspaceFolder}/morpheus", + "DISPLAY": "${localEnv:DISPLAY:-}", + "XAUTHORITY": "${localEnv:XAUTHORITY:-}", + "XDG_SESSION_TYPE": "${localEnv:XDG_SESSION_TYPE:-}", + "XDG_RUNTIME_DIR": "${localEnv:XDG_RUNTIME_DIR:-}", + "DBUS_SESSION_BUS_ADDRESS": "${localEnv:DBUS_SESSION_BUS_ADDRESS:-}" + }, + "features": { + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.6": {}, + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}, + "ghcr.io/devcontainers/features/dotnet:1": { + "version": "6.0", + "installUsingApt": false + } + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils", + "ghcr.io/devcontainers/features/docker-outside-of-docker", + "ghcr.io/devcontainers/features/dotnet" + ], + "initializeCommand": [ + "/bin/bash", + "-c", + "${localWorkspaceFolder}/.devcontainer/initialize-command.sh && mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.1-envs}" + ], + "postAttachCommand": [ + "/bin/bash", + "-c", + "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi" + ], + "workspaceFolder": "/home/coder", + "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/morpheus,type=bind,consistency=consistent", + "mounts": [ + "source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind", + "source=${localEnv:XDG_RUNTIME_DIR},target=${localEnv:XDG_RUNTIME_DIR},type=bind", + "source=/run/dbus/system_bus_socket,target=/run/dbus/system_bus_socket,type=bind", + "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind", + "source=/dev/hugepages,target=/dev/hugepages,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.1-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + ], + "customizations": { + "vscode": { + "extensions": [ + "cschlosser.doxdocgen", // Adding docstrings to C++ code + "eamodio.gitlens", // Enhanced Git support + "eeyore.yapf", // Python code formatter + "josetr.cmake-language-support-vscode", // CMake language support + "llvm-vs-code-extensions.vscode-clangd", + "llvm-vs-code-extensions.vscode-clangd", // Clangd language server for C++ + "matepek.vscode-catch2-test-adapter", + "mechatroner.rainbow-csv", // Colorizing CSV files + "mhutchie.git-graph", // Visualizing Git history and branching + "ms-azuretools.vscode-docker", // Docker support + "ms-python.debugpy", // Python debugger + "ms-python.flake8", // Python linter + "ms-python.isort", // Python import sorter + "ms-python.pylint", // Python linter + "ms-python.python", // Python language support + "ms-python.vscode-pylance", // Python language server + "ms-toolsai.jupyter", // Jupyter notebook support + "ms-vscode.cmake-tools", // CMake support for building Morpheus + "ms-vscode.cpptools", // C++ language support + "njpwerner.autodocstring", // Adding docstrings to python code + "nvidia.nsight-vscode-edition", // CUDA integration and debugging + "stkb.rewrap", // Wrapping all text in any language + "twxs.cmake", + "vadimcn.vscode-lldb", // LLDB debugger (better than GDB for C++ debugging) + "xaver.clang-format" + ], + "settings": { + "cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake", + "C_Cpp.intelliSenseEngine": "disabled", + "python.terminal.activateEnvironment": false, + "files.watcherExclude": { + "**/.git/objects/**": true, + "**/.git/subtree-cache/**": true, + "**/.cache/**": true + } + } + } + } +} \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 720df1508d..0000000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,127 +0,0 @@ -// !/bin/bash -// SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -{ - "name": "Morpheus", - "build": { - "dockerfile": "Dockerfile" - }, - - "hostRequirements": { - "gpu": true - }, - - "capAdd":[ - "SYS_NICE", - "SYS_PTRACE" - ], - - "securityOpt": [ - "seccomp=unconfined" - ], - - "runArgs": [ - "--network=morpheus" - ], - - "containerEnv": { - "HOST_MORPHEUS_ROOT": "${localWorkspaceFolder}", - "MORPHEUS_ROOT": "${containerWorkspaceFolder}", - "DEFAULT_CONDA_ENV": "morpheus", - "MAMBA_NO_BANNER": "1", - "CONDA_ALWAYS_YES": "true", - "VAULT_HOST": "https://vault.ops.k8s.rapids.ai", - "DISPLAY": "${localEnv:DISPLAY:-}", - "XAUTHORITY": "${localEnv:XAUTHORITY:-}", - "XDG_SESSION_TYPE": "${localEnv:XDG_SESSION_TYPE:-}", - "XDG_RUNTIME_DIR": "${localEnv:XDG_RUNTIME_DIR:-}", - "DBUS_SESSION_BUS_ADDRESS": "${localEnv:DBUS_SESSION_BUS_ADDRESS:-}" - }, - - "initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/initialize-command.sh" ], - - "remoteUser": "coder", - "mounts": [ - { - "type": "bind", - "source": "/tmp/.X11-unix", - "target": "/tmp/.X11-unix" - }, - { - "type": "bind", - "source": "${localEnv:XDG_RUNTIME_DIR}", - "target": "${localEnv:XDG_RUNTIME_DIR}" - }, - { - "type": "bind", - "source": "/run/dbus/system_bus_socket", - "target": "/run/dbus/system_bus_socket" - }, - { - "type":"bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock" - }, - { - "type": "bind", - "source": "${localWorkspaceFolder}/.cache/conda/envs", - "target": "/home/coder/.conda/envs" - }, - { - "type": "bind", - "source": "${localWorkspaceFolder}/../.conda/pkgs", - "target": "/home/coder/.conda/pkgs" - }, - { - "type":"bind", - "source": "${localWorkspaceFolder}/../.config", // parent folder because sister repos are sibling dirs - "target": "/home/coder/.config" - }, - { - "type": "bind", - "source": "${localWorkspaceFolder}/.devcontainer/opt/morpheus", - "target": "/opt/morpheus" - }, - - ], - - "features": { - "ghcr.io/devcontainers/features/docker-from-docker": {}, - "ghcr.io/devcontainers/features/dotnet:1": { - "version": "6.0", - "installUsingApt": false - } - }, - - "customizations": { - "vscode": { - "extensions": [ - "eamodio.gitlens", - "llvm-vs-code-extensions.vscode-clangd", - "ms-python.python", - "ms-vscode.cpptools", - "ms-vscode.cmake-tools", - "nvidia.nsight-vscode-edition", - "twxs.cmake", - "xaver.clang-format" - ], - "settings": { - "cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake", - "C_Cpp.intelliSenseEngine": "disabled", - "python.terminal.activateEnvironment": false - } - } - }, -} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index f9fdf0d858..f64e0d3b32 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3" - services: triton: @@ -29,6 +27,7 @@ services: volumes: - ${HOST_MORPHEUS_ROOT}/models:/models - ${HOST_MORPHEUS_ROOT}/examples/abp_pcap_detection/abp-pcap-xgb:/models/triton-model-repo/abp-pcap-xgb + - ${HOST_MORPHEUS_ROOT}/examples/ransomware_detection/models/ransomw-model-short-rf:/models/triton-model-repo/ransomw-model-short-rf zookeeper: image: bitnami/zookeeper:latest @@ -54,5 +53,5 @@ services: networks: default: - external: - name: morpheus + name: morpheus + external: true diff --git a/docker/optional_deps/doca.sh b/.devcontainer/docker/optional_deps/doca.sh similarity index 52% rename from docker/optional_deps/doca.sh rename to .devcontainer/docker/optional_deps/doca.sh index 97d1d108f3..f1975fbfe0 100755 --- a/docker/optional_deps/doca.sh +++ b/.devcontainer/docker/optional_deps/doca.sh @@ -19,7 +19,7 @@ set -e MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} LINUX_DISTRO=${LINUX_DISTRO:-ubuntu} LINUX_VER=${LINUX_VER:-22.04} -DOCA_VERSION=${DOCA_VERSION:-2.6.0} +DOCA_VERSION=${DOCA_VERSION:-2.7.0} # Exit early if nothing to do if [[ ${MORPHEUS_SUPPORT_DOCA} != @(TRUE|ON) ]]; then @@ -34,64 +34,21 @@ DEB_DIR=${WORKING_DIR}/deb mkdir -p ${DEB_DIR} -DOCA_REPO_LINK="https://linux.mellanox.com/public/repo/doca/${DOCA_VERSION}" -DOCA_REPO="${DOCA_REPO_LINK}/ubuntu22.04" -DOCA_REPO_ARCH="x86_64" -DOCA_UPSTREAM_REPO="${DOCA_REPO}/${DOCA_REPO_ARCH}" +DOCA_OS_VERSION="ubuntu2204" +DOCA_PKG_LINK="https://www.mellanox.com/downloads/DOCA/DOCA_v${DOCA_VERSION}/host/doca-host_${DOCA_VERSION}-204000-24.04-${DOCA_OS_VERSION}_amd64.deb" # Upgrade the base packages (diff between image and Canonical upstream repo) apt update -y apt upgrade -y -# Cleanup apt -rm -rf /var/lib/apt/lists/* -apt autoremove -y +# Install wget +apt install -y --no-install-recommends wget -# Configure DOCA Repository, and install packages -apt update -y - -# Install wget & Add the DOCA public repository -apt install -y --no-install-recommends wget software-properties-common gpg-agent -wget -qO - ${DOCA_UPSTREAM_REPO}/GPG-KEY-Mellanox.pub | apt-key add - -add-apt-repository "deb [trusted=yes] ${DOCA_UPSTREAM_REPO} ./" -apt update -y - -# Install base-rt content -apt install -y --no-install-recommends \ - doca-gpu \ - doca-gpu-dev \ - doca-prime-runtime \ - doca-prime-sdk \ - doca-sdk \ - dpcp \ - flexio \ - ibacm \ - ibverbs-utils \ - librdmacm1 \ - libibnetdisc5 \ - libibumad3 \ - libibmad5 \ - libopensm \ - libopenvswitch \ - libyara8 \ - mlnx-tools \ - ofed-scripts \ - openmpi \ - openvswitch-common \ - openvswitch-switch \ - srptools \ - mlnx-ethtool \ - mlnx-iproute2 \ - python3-pyverbs \ - rdma-core \ - ucx \ - yara - - # Cleanup apt -rm -rf /usr/lib/python3/dist-packages -apt remove -y software-properties-common gpg-agent -rm -rf /var/lib/apt/lists/* -apt autoremove -y +wget -qO - ${DOCA_PKG_LINK} -O doca-host.deb +apt install ./doca-host.deb +apt update +apt install -y doca-all +apt install -y doca-gpu doca-gpu-dev # Now install the gdrcopy library according to: https://github.com/NVIDIA/gdrcopy GDRCOPY_DIR=${WORKING_DIR}/gdrcopy diff --git a/.devcontainer/initialize-command.sh b/.devcontainer/initialize-command.sh index bdc70d1820..9741e2464a 100755 --- a/.devcontainer/initialize-command.sh +++ b/.devcontainer/initialize-command.sh @@ -16,12 +16,5 @@ set -e -# create a docker network for morpheus +# create a docker network for morpheus if it does not exist docker network inspect morpheus >/dev/null 2>&1 || docker network create morpheus - -# create the parent conda folder so it's found when mounting -mkdir -p .cache/conda/envs -mkdir -p ../.conda/pkgs - -# create a config directory if it does not exist so it's found when mounting -mkdir -p ../.config diff --git a/.devcontainer/opt/morpheus/bin/post-attach-command.sh b/.devcontainer/opt/morpheus/bin/post-attach-command.sh deleted file mode 100755 index 00a0ab3b5c..0000000000 --- a/.devcontainer/opt/morpheus/bin/post-attach-command.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Ensure our ~/.config directory has the correct permissions. If ~/.config did -# not exist, and you mount ~/.config/gh from the host, then ~/.config will be -# created with root permissions which can break things - -conda_env_find(){ - conda env list | grep "${@}" >/dev/null 2>/dev/null -} - -ENV_NAME=${ENV_NAME:-morpheus} - -sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc; - -if conda_env_find "${ENV_NAME}" ; \ -then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml --prune; \ -else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml; \ -fi diff --git a/.devcontainer/opt/morpheus/etc/.gdbinit b/.devcontainer/opt/morpheus/etc/.gdbinit deleted file mode 100644 index 115ac5e1f9..0000000000 --- a/.devcontainer/opt/morpheus/etc/.gdbinit +++ /dev/null @@ -1,3 +0,0 @@ -add-auto-load-safe-path /opt/conda - -source /opt/morpheus/etc/enable_conda_libstd_pretty_print.py diff --git a/.devcontainer/opt/morpheus/etc/enable_conda_libstd_pretty_print.py b/.devcontainer/opt/morpheus/etc/enable_conda_libstd_pretty_print.py deleted file mode 100644 index bfe973663c..0000000000 --- a/.devcontainer/opt/morpheus/etc/enable_conda_libstd_pretty_print.py +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -*- python -*- -import os -import subprocess -import sys - -import gdb - -conda_env_path = os.environ.get("CONDA_PREFIX", None) - -if (conda_env_path is not None): - - gcc_path = os.environ.get("GCC", None) - - if (gcc_path is None): - print( - "Could not find gcc from $GCC: '{}'. Ensure gxx_linux-64, gcc_linux-64, sysroot_linux-64, and gdb have been installed into the current conda environment" - .format(gcc_path)) - else: - # Get the GCC version - result = subprocess.run([gcc_path, '-dumpversion'], stdout=subprocess.PIPE) - gcc_version = result.stdout.decode("utf-8").strip() - - # Build the gcc python path - gcc_python_path = os.path.join(conda_env_path, "share", "gcc-{}".format(gcc_version), "python") - - if (os.path.exists(gcc_python_path)): - - # Add to the path for the pretty printer - sys.path.insert(0, gcc_python_path) - - # Now register the pretty printers - from libstdcxx.v6 import register_libstdcxx_printers - register_libstdcxx_printers(gdb.current_objfile()) - - print("Loaded stdlibc++ pretty printers") - else: - print("Could not find gcc python files at: {}".format(gcc_python_path)) - print( - "Ensure gxx_linux-64, gcc_linux-64, sysroot_linux-64, and gdb have been installed into the current conda environment" - ) diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml index 0ffd718b3d..34dfbd38cd 100644 --- a/.github/workflows/ci_pipe.yml +++ b/.github/workflows/ci_pipe.yml @@ -85,6 +85,7 @@ jobs: lfs: false path: 'morpheus' fetch-depth: 0 + submodules: 'recursive' - name: Get AWS credentials using OIDC uses: aws-actions/configure-aws-credentials@v1-node16 @@ -115,6 +116,7 @@ jobs: with: lfs: false path: 'morpheus' + submodules: 'recursive' - name: Get AWS credentials using OIDC uses: aws-actions/configure-aws-credentials@v1-node16 @@ -149,6 +151,7 @@ jobs: with: lfs: false path: 'morpheus' + submodules: 'recursive' - name: Get AWS credentials using OIDC uses: aws-actions/configure-aws-credentials@v1-node16 @@ -180,6 +183,9 @@ jobs: with: lfs: false path: 'morpheus' + # Fetch tags so that documentation builds for releases will report the version number correctly + fetch-tags: true + submodules: 'recursive' - name: Get AWS credentials using OIDC uses: aws-actions/configure-aws-credentials@v1-node16 @@ -213,6 +219,7 @@ jobs: lfs: false path: 'morpheus' fetch-depth: 0 + submodules: 'recursive' - name: Get AWS credentials using OIDC uses: aws-actions/configure-aws-credentials@v1-node16 diff --git a/.github/workflows/label-external-issues.yml b/.github/workflows/label-external-issues.yml index 1b13d83c54..2e71bfc801 100644 --- a/.github/workflows/label-external-issues.yml +++ b/.github/workflows/label-external-issues.yml @@ -19,27 +19,35 @@ on: issues: types: - opened - -env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} jobs: Label-Issue: runs-on: ubuntu-latest # Only run if the issue author is not part of NV-Morpheus if: ${{ ! contains(fromJSON('["OWNER", "MEMBER", "CONTRIBUTOR", "COLLABORATOR"]'), github.event.issue.author_association)}} - steps: + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: generate-token + with: + app-id: ${{ secrets.ACTIONS_APP_ID }} + private-key: ${{ secrets.ACTIONS_APP_KEY }} + - name: add-triage-label + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | issue_url=${{ github.event.issue.html_url }} gh issue edit ${issue_url} --add-label "Needs Triage,external" - + - name: add-comment-to-issue + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | issue_url=${{ github.event.issue.html_url }} author=${{ github.event.issue.user.login }} echo ${author} gh issue comment ${issue_url} --body "Hi @${author}! - + Thanks for submitting this issue - our team has been notified and we'll get back to you as soon as we can! - In the mean time, feel free to add any relevant information to this issue." + In the meantime, feel free to add any relevant information to this issue." diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e928fafd7f..f135ff2dc2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -43,14 +43,6 @@ permissions: jobs: - pr-builder: - needs: - - prepare - - checks - - ci_pipe - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 - prepare: # Executes the get-pr-info action to determine if the PR has the skip-ci label, if the action fails we assume the # PR does not have the label @@ -95,7 +87,17 @@ jobs: # Disable conda upload for now, once we have morpheus packages in conda forge set the value to # !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' conda_upload_label: "" - container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-240221 - test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-240221 + container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-240614 + test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-240614 secrets: NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + + pr-builder: + # Always run this step even if others are skipped or cancelled + if: '!cancelled()' + needs: + - prepare + - checks + - ci_pipe + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 diff --git a/.github/workflows/project_automation_set_in_progress.yml b/.github/workflows/project_automation_set_in_progress.yml index 77d2bd8f17..414c013cb3 100644 --- a/.github/workflows/project_automation_set_in_progress.yml +++ b/.github/workflows/project_automation_set_in_progress.yml @@ -23,33 +23,33 @@ on: pull_request_review: # Run this action when a PR is reviewed types: [submitted] - + env: - ORG: ${{ github.event.repository.owner.login }} - PR_NUMBER: ${{ github.event.pull_request.number }} - REPO: ${{ github.event.repository.name }} - PR_GLOBAL_ID: ${{ github.event.pull_request.node_id}} + ORG: ${{ github.event.repository.owner.login }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.event.repository.name }} + PR_GLOBAL_ID: ${{ github.event.pull_request.node_id}} - # The environment vars below are hard-coded from external queries to save time + complexity here - # Note: PVT means Project V2, not "Private" - # PVT = Project V2, PVTSSF = Project V2 Single Select Field, PVTIF = Project V2 Iteration Field - PROJECT_ID: "PVT_kwDOBkAsks4ACeio" - STATUS_FIELD_ID: "PVTSSF_lADOBkAsks4ACeiozgBbno0" - WORKING_SPRINT_FIELD_ID: "PVTIF_lADOBkAsks4ACeiozgJ_KCY" - START_SPRINT_FIELD_ID: "PVTIF_lADOBkAsks4ACeiozgI90p0" - IN_PROGRESS_PROJECT_OPTION_ID: "47fc9ee4" - IN_REVIEW_PROJECT_OPTION_ID: "eb7a6302" + # The environment vars below are hard-coded from external queries to save time + complexity here + # Note: PVT means Project V2, not "Private" + # PVT = Project V2, PVTSSF = Project V2 Single Select Field, PVTIF = Project V2 Iteration Field + PROJECT_ID: "PVT_kwDOBkAsks4ACeio" + STATUS_FIELD_ID: "PVTSSF_lADOBkAsks4ACeiozgBbno0" + WORKING_SPRINT_FIELD_ID: "PVTIF_lADOBkAsks4ACeiozgJ_KCY" + START_SPRINT_FIELD_ID: "PVTIF_lADOBkAsks4ACeiozgI90p0" + IN_PROGRESS_PROJECT_OPTION_ID: "47fc9ee4" + IN_REVIEW_PROJECT_OPTION_ID: "eb7a6302" jobs: project_automation_in_progress: runs-on: ubuntu-latest # We use the default GitHub token to perform the draft update because - # The pull_request_review trigger does not grant access to the secrets.PROJECT_MANAGEMENT_PAT + # The pull_request_review trigger does not grant access to secrets permissions: issues: write pull-requests: write - + steps: - name: Check if changes requested from a reviewer id: check_changes_requested @@ -58,7 +58,7 @@ jobs: if [ ${{ github.event.review.state }} != 'changes_requested' ]; then echo "Changes not requested, exiting" exit 0 - + # If it is requesting changes, set PR to draft # We use the default token here since we're granting write access to the PR elif [ ${{ github.event.pull_request.draft }} == false ]; then @@ -80,8 +80,6 @@ jobs: - name: Select Status Field Value id: select_status_field_value if: github.event_name == 'pull_request_target' - env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} run: | # If it's not a draft and it's an opened trigger, the status should be "Ready for Review", otherwise "In Progress" if [ ${{ github.event.pull_request.draft }} == false ] && [ ${{ github.event.action }} == "opened" ]; then @@ -96,8 +94,6 @@ jobs: - name: Get PR Project ID id: get_pr_id if: github.event_name == 'pull_request_target' - env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} run: | # Query up to 10 projects for the PR gh api graphql -f query=' @@ -122,7 +118,7 @@ jobs: } } }' > project_data.json - + # Filter the json result to only the project-specific ID for the PR # A PR can be in multiple projects so we need to filter by the project ID we want pr_id=$(jq -r '.data.organization.repository.issueOrPullRequest.projectItems.edges[] | @@ -130,12 +126,10 @@ jobs: .node.id' project_data.json) echo "PR_ID=$pr_id" >> $GITHUB_ENV continue-on-error: true - + - name: Set PR Fields id: set_pr_fields if: github.event_name == 'pull_request_target' - env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} run: | gh api graphql -f query=' mutation { @@ -154,7 +148,7 @@ jobs: } } }' - + # Check if the PR has a start sprint assigned, save the result for the linked issues gh api graphql -f query=' query { @@ -171,7 +165,7 @@ jobs: }' > start_sprint_exists_data.json start_sprint_option_id=$(jq -r '.data.node.fieldValueByName.id' start_sprint_exists_data.json) echo "START_SPRINT_OPTION_ID=$start_sprint_option_id" >> $GITHUB_ENV - + # If there is no start sprint assigned, assign the current start sprint if [ "$start_sprint_option_id" == 'null' ]; then # Get current start sprint iteration id @@ -197,7 +191,7 @@ jobs: }' > start_sprint_option_data.json current_start_sprint_option_id=$(jq -r '.data.node.field.configuration.iterations[0].id' start_sprint_option_data.json) echo "CURRENT_START_SPRINT_OPTION_ID=$current_start_sprint_option_id" >> $GITHUB_ENV - + # The query below is constructed differently than the ones above due to bash variable syntax + github actions syntax interactions QUERY="mutation { updateProjectV2ItemFieldValue( @@ -217,7 +211,7 @@ jobs: }" gh api graphql --field query="$QUERY" fi - + # Assign the current working sprint to the PR (faster/simpler to just overwrite even if it is the same) gh api graphql -f query=' query { @@ -240,7 +234,7 @@ jobs: }' > working_sprint_options_data.json current_working_sprint_option_id=$(jq -r '.data.node.field.configuration.iterations[0].id' working_sprint_options_data.json) echo "CURRENT_WORKING_SPRINT_OPTION_ID=$current_working_sprint_option_id" >> $GITHUB_ENV - + # Set the working sprint to the current working sprint QUERY="mutation { updateProjectV2ItemFieldValue( @@ -260,12 +254,10 @@ jobs: }" gh api graphql --field query="$QUERY" continue-on-error: true - + - name: Sync Linked Issues id: sync_linked_issues if: github.event_name == 'pull_request_target' - env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} run: | # Find the linked issues to the PR gh api graphql -f query=' @@ -303,7 +295,7 @@ jobs: issue_ids=$(jq -r '.data.organization.repository.issueOrPullRequest.closingIssuesReferences.edges[].node.projectItems.edges[] | select(.node.project.id == "${{ env.PROJECT_ID }}") | .node.id' linked_issues.json) - + # For each linked issue, set the status to "In Progress", the Working Sprint to the current working sprint # If there's no Start Sprint, set that to the current Start Sprint as well for issue_id in $issue_ids; do diff --git a/.github/workflows/project_automation_set_in_review.yml b/.github/workflows/project_automation_set_in_review.yml index bacb6812df..9734a50f85 100644 --- a/.github/workflows/project_automation_set_in_review.yml +++ b/.github/workflows/project_automation_set_in_review.yml @@ -20,7 +20,7 @@ on: # Run this action when a PR is opened or edited # Issues do not have a graphQL connection to linked PRs so we can't use that event types: [ready_for_review, review_requested] - + env: ORG: ${{ github.event.repository.owner.login }} PR_NUMBER: ${{ github.event.pull_request.number }} @@ -36,16 +36,23 @@ env: jobs: project_automation_in_review: runs-on: ubuntu-latest - + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: generate-token + with: + app-id: ${{ secrets.ACTIONS_APP_ID }} + private-key: ${{ secrets.ACTIONS_APP_KEY }} + - name: Wait 1 Second id: sleep run: sleep 1 - + - name: Get PR Project ID id: get_pr_id env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Query up to 10 projects for the PR gh api graphql -f query=' @@ -70,7 +77,7 @@ jobs: } } }' > project_data.json - + # Filter the json result to only the project-specific ID for the PR # A PR can be in multiple projects so we need to filter by the project ID we want pr_id=$(jq -r '.data.organization.repository.issueOrPullRequest.projectItems.edges[] | @@ -81,8 +88,8 @@ jobs: - name: Set PR to In Review id: set_pr_in_review - env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Update the PR status to In Review gh api graphql -f query=' @@ -107,7 +114,7 @@ jobs: - name: Set Linked Issues to In Review id: update_linked_issues env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | gh api graphql -f query=' query { @@ -144,7 +151,7 @@ jobs: issue_ids=$(jq -r '.data.organization.repository.issueOrPullRequest.closingIssuesReferences.edges[].node.projectItems.edges[] | select(.node.project.id == "${{ env.PROJECT_ID }}") | .node.id' linked_issue_data.json) - + # Set Linked Issues to In Review for issue_id in $issue_ids; do # The query below is constructed differently than the others due to bash variable syntax + github actions syntax interactions diff --git a/.github/workflows/project_automation_set_roadmap.yml b/.github/workflows/project_automation_set_roadmap.yml index 19d458908f..2759e5b943 100644 --- a/.github/workflows/project_automation_set_roadmap.yml +++ b/.github/workflows/project_automation_set_roadmap.yml @@ -22,7 +22,7 @@ on: issues: # Run this action when an issue is closed types: [closed] - + env: ORG: ${{ github.event.repository.owner.login }} PR_NUMBER: ${{ github.event.pull_request.number }} # evaluates to null for issues @@ -38,13 +38,20 @@ env: jobs: project_automation_roadmap_on_close: runs-on: ubuntu-latest - - steps: + + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: generate-token + with: + app-id: ${{ secrets.ACTIONS_APP_ID }} + private-key: ${{ secrets.ACTIONS_APP_KEY }} + - name: Get PR Project ID if: github.event_name == 'pull_request_target' id: get_pr_id env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Query up to 10 projects for the PR gh api graphql -f query=' @@ -69,7 +76,7 @@ jobs: } } }' > project_data.json - + # Filter the json result to only the project-specific ID for the PR # A PR can be in multiple projects so we need to filter by the project ID we want pr_id=$(jq -r '.data.organization.repository.issueOrPullRequest.projectItems.edges[] | @@ -77,12 +84,12 @@ jobs: .node.id' project_data.json) echo "ITEM_ID=$pr_id" >> $GITHUB_ENV continue-on-error: true - + - name: Get Issue Project ID if: github.event_name == 'issues' id: get_issue_id env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Query up to 10 projects for the Issue gh api graphql -f query=' @@ -107,7 +114,7 @@ jobs: } } }' > project_data.json - + # Filter the json result to only the project-specific ID for the PR # A PR can be in multiple projects so we need to filter by the project ID we want issue_id=$(jq -r '.data.organization.repository.issueOrPullRequest.projectItems.edges[] | @@ -119,8 +126,8 @@ jobs: - name: Get Current Release id: get_current_release env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} - run: | + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + run: | # Get current roadmap id # We maintain the roadmap as a single select field in the project, with the first value being the upcoming release @@ -146,8 +153,8 @@ jobs: - name: Set Item Roadmap id: set_item_roadmap - env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Perform the mutation to update the Roadmap gh api graphql -f query=' diff --git a/.github/workflows/project_automation_sync_pr_issues.yml b/.github/workflows/project_automation_sync_pr_issues.yml index 5dfa479138..7533c2e512 100644 --- a/.github/workflows/project_automation_sync_pr_issues.yml +++ b/.github/workflows/project_automation_sync_pr_issues.yml @@ -38,6 +38,13 @@ jobs: runs-on: ubuntu-latest steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: generate-token + with: + app-id: ${{ secrets.ACTIONS_APP_ID }} + private-key: ${{ secrets.ACTIONS_APP_KEY }} + - name: Wait 1 Second id: sleep run: sleep 1 @@ -45,7 +52,7 @@ jobs: - name: Get PR Project ID id: get_pr_id env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Query up to 10 projects for the PR gh api graphql -f query=' @@ -70,7 +77,7 @@ jobs: } } }' > project_data.json - + # Filter the json result to only the project-specific ID for the PR # A PR can be in multiple projects so we need to filter by the project ID we want pr_id=$(jq -r '.data.organization.repository.issueOrPullRequest.projectItems.edges[] | @@ -81,8 +88,8 @@ jobs: - name: Get Status, Start Sprint, and Working Sprint Fields from the PR id: get_pr_status - env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Query the PR's status from the project gh api graphql -f query=' @@ -140,7 +147,7 @@ jobs: - name: Sync Linked Issues id: update_linked_issues env: - GITHUB_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | gh api graphql -f query=' query { diff --git a/.github/workflows/project_automation_track_pr_dates.yml b/.github/workflows/project_automation_track_pr_dates.yml index d660e96484..28a05e4f3f 100644 --- a/.github/workflows/project_automation_track_pr_dates.yml +++ b/.github/workflows/project_automation_track_pr_dates.yml @@ -24,7 +24,6 @@ env: ORG: ${{ github.event.repository.owner.login }} PR_NUMBER: ${{ github.event.pull_request.number }} REPO: ${{ github.event.repository.name }} - GH_TOKEN: ${{ secrets.PROJECT_MANAGEMENT_PAT }} OPENED_DATE: ${{ github.event.pull_request.created_at }} CLOSED_DATE: ${{ github.event.pull_request.closed_at }} TRIGGER: ${{ github.event.action }} @@ -42,12 +41,21 @@ jobs: runs-on: ubuntu-latest steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: generate-token + with: + app-id: ${{ secrets.ACTIONS_APP_ID }} + private-key: ${{ secrets.ACTIONS_APP_KEY }} + - name: Sleep 1s id: sleep_1s run: sleep 1 - name: Get PR Project ID id: get_pr_id + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Query up to 10 projects for the PR gh api graphql -f query=' @@ -72,7 +80,7 @@ jobs: } } }' > project_data.json - + # Filter the json result to only the project-specific ID for the PR # A PR can be in multiple projects so we need to filter by the project ID we want pr_id=$(jq -r '.data.organization.repository.issueOrPullRequest.projectItems.edges[] | @@ -84,6 +92,8 @@ jobs: - name: Get Date for Review Start if: ${{ env.TRIGGER == 'ready_for_review' }} || (${{ env.TRIGGER == 'opened' }} && ${{ github.event.pull_request.draft == false }}) id: get_date + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Get the current date in the format required by ProjectsV2 GraphQL # Uses EST timezone @@ -99,6 +109,8 @@ jobs: - name: Set PR Opened Date id: set_open_date if: ${{ env.TRIGGER == 'opened' }} + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Mutation to update the PRs's Opened Date field echo "${{ env.OPENED_DATE }}" @@ -125,7 +137,9 @@ jobs: - name: Set PR Review Start Date id: set_review_date if: ${{ env.TRIGGER == 'ready_for_review' }} || (${{ env.TRIGGER == 'opened' }} && ${{ github.event.pull_request.draft == false }}) - run: | + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + run: | # Check the Review Start Date gh api graphql -f query=' query { @@ -168,6 +182,8 @@ jobs: - name: Set PR Closed Date id: set_closed_date if: ${{ env.TRIGGER == 'closed' }} + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} run: | # Mutation to update the PR's Closed Date field gh api graphql -f query=' diff --git a/.gitmodules b/.gitmodules index a92915a355..2059804fe4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,8 @@ [submodule "external/morpheus-visualizations"] path = external/morpheus-visualizations url = https://github.com/nv-morpheus/morpheus-visualizations.git - branch = branch-24.03 + branch = branch-24.06 [submodule "external/utilities"] path = external/utilities url = https://github.com/nv-morpheus/utilities.git - branch = branch-24.03 + branch = branch-24.06 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2054c813d..2682c98fb7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.7.1 + rev: v1.13.11 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/CHANGELOG.md b/CHANGELOG.md index ef1d9eff16..67b1cc181d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,93 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> +# Morpheus 24.06.00 (03 Jul 2024) +## 🚨 Breaking Changes + +- Introduce multi-endpoint servers and health check endpoints to HttpServerSourceStage ([#1734](https://github.com/nv-morpheus/Morpheus/pull/1734)) [@jadu-nv](https://github.com/jadu-nv) +- Update devcontainer to use latest build utils ([#1658](https://github.com/nv-morpheus/Morpheus/pull/1658)) [@cwharris](https://github.com/cwharris) +- Update CI to install DOCA and build Morpheus DOCA components. ([#1622](https://github.com/nv-morpheus/Morpheus/pull/1622)) [@cwharris](https://github.com/cwharris) +- Support non-json serializable objects in LLMContext ([#1589](https://github.com/nv-morpheus/Morpheus/pull/1589)) [@dagardner-nv](https://github.com/dagardner-nv) + +## 🐛 Bug Fixes + +- Fix LLM Agents Kafka pipeline ([#1793](https://github.com/nv-morpheus/Morpheus/pull/1793)) [@dagardner-nv](https://github.com/dagardner-nv) +- Add fetch data command in build_container_release.sh ([#1787](https://github.com/nv-morpheus/Morpheus/pull/1787)) [@ifengw-nv](https://github.com/ifengw-nv) +- Add cuda and cudf to link targets for C++ examples ([#1777](https://github.com/nv-morpheus/Morpheus/pull/1777)) [@dagardner-nv](https://github.com/dagardner-nv) +- Release container fixes ([#1766](https://github.com/nv-morpheus/Morpheus/pull/1766)) [@dagardner-nv](https://github.com/dagardner-nv) +- Remove setting of `prog_name`, this implies that an executable named `morpheus_llm` exists ([#1759](https://github.com/nv-morpheus/Morpheus/pull/1759)) [@dagardner-nv](https://github.com/dagardner-nv) +- Provide a default set of questions for the standalone RAG pipeline ([#1758](https://github.com/nv-morpheus/Morpheus/pull/1758)) [@dagardner-nv](https://github.com/dagardner-nv) +- Disable shared memory by default, and fix `--stop_after` flag for `vdb_upload` example ([#1755](https://github.com/nv-morpheus/Morpheus/pull/1755)) [@dagardner-nv](https://github.com/dagardner-nv) +- Fix conda errors in release container ([#1750](https://github.com/nv-morpheus/Morpheus/pull/1750)) [@dagardner-nv](https://github.com/dagardner-nv) +- Fixes for C++ impl for `DeserializeStage` and add missing `get_info` overloads to `SlicedMessageMeta` ([#1749](https://github.com/nv-morpheus/Morpheus/pull/1749)) [@dagardner-nv](https://github.com/dagardner-nv) +- Add GIT_CLONE_PROTECTION_ACTIVE env config to fix build script ([#1748](https://github.com/nv-morpheus/Morpheus/pull/1748)) [@jadu-nv](https://github.com/jadu-nv) +- Fix triton multi threading when using the C++ stage ([#1739](https://github.com/nv-morpheus/Morpheus/pull/1739)) [@mdemoret-nv](https://github.com/mdemoret-nv) +- resolve rapids-dependency-file-generator warning ([#1735](https://github.com/nv-morpheus/Morpheus/pull/1735)) [@jameslamb](https://github.com/jameslamb) +- Updating all uses of the `secrets.PROJECT_MANAGEMENT_PAT` to use a registered Github App ([#1730](https://github.com/nv-morpheus/Morpheus/pull/1730)) [@mdemoret-nv](https://github.com/mdemoret-nv) +- DOCA: fix optional deps + remove PreallocatorMixin from source stage ([#1729](https://github.com/nv-morpheus/Morpheus/pull/1729)) [@e-ago](https://github.com/e-ago) +- Remove `pyarrow_hotfix` import from `__init__.py` ([#1692](https://github.com/nv-morpheus/Morpheus/pull/1692)) [@efajardo-nv](https://github.com/efajardo-nv) +- Support the filter_null parameter in the C++ impl of the FileSourceStage ([#1689](https://github.com/nv-morpheus/Morpheus/pull/1689)) [@dagardner-nv](https://github.com/dagardner-nv) +- Enable C++ mode for `abp_pcap_detection` example ([#1687](https://github.com/nv-morpheus/Morpheus/pull/1687)) [@dagardner-nv](https://github.com/dagardner-nv) +- Strip HTML & XML tags from RSS feed input ([#1670](https://github.com/nv-morpheus/Morpheus/pull/1670)) [@dagardner-nv](https://github.com/dagardner-nv) +- Truncate strings exceeding max_length when inserting to Milvus ([#1665](https://github.com/nv-morpheus/Morpheus/pull/1665)) [@dagardner-nv](https://github.com/dagardner-nv) +- Fix a typo in the devcontainer base image ([#1638](https://github.com/nv-morpheus/Morpheus/pull/1638)) [@cwharris](https://github.com/cwharris) +- Fix tests to detect issue #1626 ([#1629](https://github.com/nv-morpheus/Morpheus/pull/1629)) [@dagardner-nv](https://github.com/dagardner-nv) +- Fix `cupy_to_tensor` to also infer `uint8` and `int8` dtypes ([#1621](https://github.com/nv-morpheus/Morpheus/pull/1621)) [@efajardo-nv](https://github.com/efajardo-nv) +- Add struct column support to `cudf_helpers` ([#1538](https://github.com/nv-morpheus/Morpheus/pull/1538)) [@efajardo-nv](https://github.com/efajardo-nv) + +## 📖 Documentation + +- Cleanup docs so that each as a single H1 title ([#1794](https://github.com/nv-morpheus/Morpheus/pull/1794)) [@dagardner-nv](https://github.com/dagardner-nv) +- Mark the LLM Agents Kafka pipeline as broken ([#1792](https://github.com/nv-morpheus/Morpheus/pull/1792)) [@dagardner-nv](https://github.com/dagardner-nv) +- Document supported environments for each example ([#1786](https://github.com/nv-morpheus/Morpheus/pull/1786)) [@dagardner-nv](https://github.com/dagardner-nv) +- Removes unused environment variables from Morpheus build docs ([#1784](https://github.com/nv-morpheus/Morpheus/pull/1784)) [@yczhang-nv](https://github.com/yczhang-nv) +- Remove documentation for yaml config files in `vdb_upload` pipeline until #1752 is resolved ([#1778](https://github.com/nv-morpheus/Morpheus/pull/1778)) [@dagardner-nv](https://github.com/dagardner-nv) +- Remove out of date instructions from `contributing.md` ([#1774](https://github.com/nv-morpheus/Morpheus/pull/1774)) [@dagardner-nv](https://github.com/dagardner-nv) +- Add troubleshooting reference for unsuccessful documentation builds ([#1768](https://github.com/nv-morpheus/Morpheus/pull/1768)) [@ifengw-nv](https://github.com/ifengw-nv) +- Remove pre-built container section from `getting_started.md` ([#1764](https://github.com/nv-morpheus/Morpheus/pull/1764)) [@yczhang-nv](https://github.com/yczhang-nv) +- Clarify Documentation: Run fetch_data.py Outside Docker Container ([#1762](https://github.com/nv-morpheus/Morpheus/pull/1762)) [@ifengw-nv](https://github.com/ifengw-nv) +- Add function return documentation for `LLMService` ([#1721](https://github.com/nv-morpheus/Morpheus/pull/1721)) [@acaklovic-nv](https://github.com/acaklovic-nv) +- Fix description for `cache_mode` option of DFP Rolling Window module ([#1707](https://github.com/nv-morpheus/Morpheus/pull/1707)) [@efajardo-nv](https://github.com/efajardo-nv) +- Update root-cause-analysis-model-card.md ([#1684](https://github.com/nv-morpheus/Morpheus/pull/1684)) [@HesAnEasyCoder](https://github.com/HesAnEasyCoder) +- Update abp-model-card.md ([#1683](https://github.com/nv-morpheus/Morpheus/pull/1683)) [@HesAnEasyCoder](https://github.com/HesAnEasyCoder) +- Update dfp-model-card.md ([#1682](https://github.com/nv-morpheus/Morpheus/pull/1682)) [@HesAnEasyCoder](https://github.com/HesAnEasyCoder) +- Update gnn-fsi-model-card.md ([#1681](https://github.com/nv-morpheus/Morpheus/pull/1681)) [@HesAnEasyCoder](https://github.com/HesAnEasyCoder) +- Update phishing-model-card.md ([#1680](https://github.com/nv-morpheus/Morpheus/pull/1680)) [@HesAnEasyCoder](https://github.com/HesAnEasyCoder) +- Update examples to execute from the root of the repo ([#1674](https://github.com/nv-morpheus/Morpheus/pull/1674)) [@dagardner-nv](https://github.com/dagardner-nv) +- Update dfp-model-card.md ([#1644](https://github.com/nv-morpheus/Morpheus/pull/1644)) [@HesAnEasyCoder](https://github.com/HesAnEasyCoder) + +## 🚀 New Features + +- Add ransomware model to devcontainer ([#1785](https://github.com/nv-morpheus/Morpheus/pull/1785)) [@yczhang-nv](https://github.com/yczhang-nv) +- Introduce multi-endpoint servers and health check endpoints to HttpServerSourceStage ([#1734](https://github.com/nv-morpheus/Morpheus/pull/1734)) [@jadu-nv](https://github.com/jadu-nv) +- Support `ControlMessage` for `Preprocess` and `PostProcess` stages ([#1623](https://github.com/nv-morpheus/Morpheus/pull/1623)) [@yczhang-nv](https://github.com/yczhang-nv) +- Update CI to install DOCA and build Morpheus DOCA components. ([#1622](https://github.com/nv-morpheus/Morpheus/pull/1622)) [@cwharris](https://github.com/cwharris) +- DOCA stage split: source + convert ([#1617](https://github.com/nv-morpheus/Morpheus/pull/1617)) [@e-ago](https://github.com/e-ago) +- `ControlMessage` support in `TritonInferenceStage` and `PreallocatorMixin` ([#1610](https://github.com/nv-morpheus/Morpheus/pull/1610)) [@cwharris](https://github.com/cwharris) + +## 🛠️ Improvements + +- Merge Agent Morpheus changes ([#1760](https://github.com/nv-morpheus/Morpheus/pull/1760)) [@dagardner-nv](https://github.com/dagardner-nv) +- Fix `RabbitMQ` output path ([#1756](https://github.com/nv-morpheus/Morpheus/pull/1756)) [@yczhang-nv](https://github.com/yczhang-nv) +- Misc improvements for sid_visualization example ([#1751](https://github.com/nv-morpheus/Morpheus/pull/1751)) [@dagardner-nv](https://github.com/dagardner-nv) +- Auditing the dependencies and syncing `dependencies.yaml` with `meta.yaml` ([#1743](https://github.com/nv-morpheus/Morpheus/pull/1743)) [@mdemoret-nv](https://github.com/mdemoret-nv) +- Always run the PR builder step even if others are cancelled ([#1736](https://github.com/nv-morpheus/Morpheus/pull/1736)) [@mdemoret-nv](https://github.com/mdemoret-nv) +- Enable Python install by default in `compile.sh` ([#1724](https://github.com/nv-morpheus/Morpheus/pull/1724)) [@dagardner-nv](https://github.com/dagardner-nv) +- Generate deprecation warning for `MultiMessage` ([#1719](https://github.com/nv-morpheus/Morpheus/pull/1719)) [@yczhang-nv](https://github.com/yczhang-nv) +- Improve the logging tests and add support for resetting the logger ([#1716](https://github.com/nv-morpheus/Morpheus/pull/1716)) [@mdemoret-nv](https://github.com/mdemoret-nv) +- Allow passing `metadata` to `LangChainAgentNode._run_single` ([#1710](https://github.com/nv-morpheus/Morpheus/pull/1710)) [@dagardner-nv](https://github.com/dagardner-nv) +- Support passing a custom parser to `HttpServerSourceStage` and `HttpClientSourceStage` stages ([#1705](https://github.com/nv-morpheus/Morpheus/pull/1705)) [@dagardner-nv](https://github.com/dagardner-nv) +- Use EnvConfigValue for passing env-configured arguments to services ([#1704](https://github.com/nv-morpheus/Morpheus/pull/1704)) [@cwharris](https://github.com/cwharris) +- Remove unused MLflow client arg from DFP inference implementations ([#1700](https://github.com/nv-morpheus/Morpheus/pull/1700)) [@efajardo-nv](https://github.com/efajardo-nv) +- Add group by column stage ([#1699](https://github.com/nv-morpheus/Morpheus/pull/1699)) [@dagardner-nv](https://github.com/dagardner-nv) +- Fix non-deterministic output of gnn sampler ([#1677](https://github.com/nv-morpheus/Morpheus/pull/1677)) [@tzemicheal](https://github.com/tzemicheal) +- Ensuring consistent use of the export macro `MORPHEUS_EXPORT` ([#1672](https://github.com/nv-morpheus/Morpheus/pull/1672)) [@aserGarcia](https://github.com/aserGarcia) +- Update devcontainer to use latest build utils ([#1658](https://github.com/nv-morpheus/Morpheus/pull/1658)) [@cwharris](https://github.com/cwharris) +- Update `ControlMessage` to hold arbitrary Python objects & update `MessageMeta` to copy & slice ([#1637](https://github.com/nv-morpheus/Morpheus/pull/1637)) [@yczhang-nv](https://github.com/yczhang-nv) +- Use conda env create --yes instead of --force ([#1636](https://github.com/nv-morpheus/Morpheus/pull/1636)) [@efajardo-nv](https://github.com/efajardo-nv) +- Misc CI improvements ([#1618](https://github.com/nv-morpheus/Morpheus/pull/1618)) [@dagardner-nv](https://github.com/dagardner-nv) +- Support non-json serializable objects in LLMContext ([#1589](https://github.com/nv-morpheus/Morpheus/pull/1589)) [@dagardner-nv](https://github.com/dagardner-nv) + # Morpheus 24.03.02 (24 Apr 2024) ## 🐛 Bug Fixes diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b57d23b1b..f2658237ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,7 +92,7 @@ morpheus_utils_initialize_cuda_arch(morpheus) # Note intentionally excluding CUDA from the LANGUAGES list allowing us to set some clang specific settings later when # we call morpheus_utils_enable_cuda() project(morpheus - VERSION 24.03.00 + VERSION 24.06.00 LANGUAGES C CXX ) diff --git a/ci/check_style.sh b/ci/check_style.sh index beb561bb4f..66a99f00d6 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -13,7 +13,7 @@ rapids-logger "Create checks conda environment" rapids-dependency-file-generator \ --output conda \ - --file_key checks \ + --file-key checks \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n checks diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index 963c7e4729..2d3921bbe9 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -58,53 +58,60 @@ outputs: - pkg-config =0.29 # for mrc cmake - sysroot_linux-64 =2.17 host: + # CUDA dependencies - cuda-cudart-dev {{ cuda_compiler_version }}.* - cuda-libraries-dev {{ cuda_compiler_version }}.* # required by matx for libcublas-dev - cuda-nvml-dev {{ cuda_compiler_version }}.* - cuda-nvrtc-dev {{ cuda_compiler_version }}.* - cuda-nvtx-dev {{ cuda_compiler_version }}.* - cuda-version {{ cuda_compiler_version }}.* + # Non-CUDA dependencies - cudf {{ rapids_version }} - - cython =3.0 + - cython 3.0.* + - glog 0.6.* - libcudf {{ rapids_version }} - librdkafka >=1.9.2,<1.10.0a0 - mrc {{ minor_version }} + - nlohmann_json 3.11.* - pip - - pybind11-stubgen =0.10.5 + - pybind11-stubgen 0.10.5 - python {{ python }} - - rapidjson =1.1.0 + - rapidjson 1.1.0 - rdma-core >=48 # Needed for DOCA. - - scikit-build =0.17.6 + - scikit-build 0.17.6 - versioneer-518 - - zlib # required to build triton client + - zlib 1.2.13 # required to build triton client run: # Runtime only requirements. This + setup.py is the definitive runtime requirement list + # This should be synced with `runtime` in dependencies.yaml + - appdirs + - beautifulsoup4 - click >=8 - - configargparse =1.5.* - cuda-version {{ cuda_compiler_version }}.* - cudf - cupy # Version determined from cudf - datacompy =0.10.* - - dill - - distributed + - dill =0.3.7 - docker-py =5.0.* - - grpcio # Version determined from cudf - - libmrc + - elasticsearch ==8.9.0 + - feedparser =6.0.* + - grpcio =1.59.* - libwebp>=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863 - mlflow>=2.10.0,<3 - mrc - - networkx>=2.8 + - networkx=2.8.8 - numpydoc =1.5.* - nvtabular =23.08.00 - - pandas + - pydantic - pluggy =1.3.* - - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF - python - python-confluent-kafka >=1.9.2,<1.10.0a0 - - pytorch + - python-graphviz - pytorch-cuda + - pytorch * *cuda* + - rapids-dask-dependency {{ rapids_version }} # provides dask and distributed - requests - - requests-cache =1.1 + - requests-cache =1.1.* - scikit-learn =1.3.2.* - sqlalchemy <2.0 # 2.0 is incompatible with pandas=1.3 - tqdm =4.* diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh index d53ce56323..6c8cd3183d 100755 --- a/ci/conda/recipes/run_conda_build.sh +++ b/ci/conda/recipes/run_conda_build.sh @@ -19,7 +19,7 @@ NUMARGS=$# ARGS=$* function hasArg { - (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") + (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") } function get_version() { @@ -99,6 +99,14 @@ CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvid CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}pytorch") CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}defaults") +if [[ ${NUMARGS} == 0 ]]; then + echo -e "${r}ERROR: No arguments were provided. Please provide at least one package to build. Available packages:${x}" + echo -e "${r} morpheus${x}" + echo -e "${r} pydebug${x}" + echo -e "${r}Exiting...${x}" + exit 12 +fi + if hasArg morpheus; then # Set GIT_VERSION to set the project version inside of meta.yaml export GIT_VERSION="$(get_version)" @@ -110,10 +118,10 @@ if hasArg morpheus; then fi if hasArg pydebug; then - export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2) + export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2) - echo "Running conda-build for python-dbg..." - set -x - conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ./ci/conda/recipes/python-dbg - set +x + echo "Running conda-build for python-dbg..." + set -x + conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ./ci/conda/recipes/python-dbg + set +x fi diff --git a/ci/runner/Dockerfile b/ci/runner/Dockerfile index 57cc66f666..40b035c402 100644 --- a/ci/runner/Dockerfile +++ b/ci/runner/Dockerfile @@ -1,5 +1,3 @@ -# syntax=docker/dockerfile:1.3 - # SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -37,6 +35,11 @@ SHELL ["/bin/bash", "-c"] # Create conda environment COPY ./dependencies.yaml /tmp/conda/ +RUN <=1.13.*" +conda clean -aipty +EOF # ============ build ================== FROM base as build @@ -49,13 +52,27 @@ ARG PYTHON_VER RUN rapids-dependency-file-generator \ --config /tmp/conda/dependencies.yaml \ --output conda \ - --file_key build \ + --file-key build \ --matrix "cuda=${CUDA_SHORT_VER};arch=${ARCH};py=${PYTHON_VER}" > /tmp/conda/env.yaml && \ CONDA_ALWAYS_YES=true /opt/conda/bin/conda env create -n ${PROJ_NAME} -q --file /tmp/conda/env.yaml && \ sed -i "s/conda activate base/conda activate ${PROJ_NAME}/g" ~/.bashrc && \ conda clean -afy && \ rm -rf /tmp/conda +ENV MORPHEUS_SUPPORT_DOCA=ON + +COPY ./.devcontainer/docker/optional_deps/doca.sh /tmp/doca/ + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \ + apt install --no-install-recommends -y \ + build-essential \ + libtool \ + automake && \ + apt clean && \ + /tmp/doca/doca.sh /tmp/doca && \ + rm -rf /tmp/doca + # ============ test ================== FROM base as test @@ -76,7 +93,7 @@ RUN apt update && \ RUN rapids-dependency-file-generator \ --config /tmp/conda/dependencies.yaml \ --output conda \ - --file_key test \ + --file-key test \ --matrix "cuda=${CUDA_SHORT_VER};arch=${ARCH};py=${PYTHON_VER}" > /tmp/conda/env.yaml && \ CONDA_ALWAYS_YES=true /opt/conda/bin/conda env create -n ${PROJ_NAME} -q --file /tmp/conda/env.yaml && \ sed -i "s/conda activate base/conda activate ${PROJ_NAME}/g" ~/.bashrc && \ diff --git a/ci/scripts/bootstrap_local_ci.sh b/ci/scripts/bootstrap_local_ci.sh index 3051b13af1..45c68b3ae4 100755 --- a/ci/scripts/bootstrap_local_ci.sh +++ b/ci/scripts/bootstrap_local_ci.sh @@ -25,6 +25,8 @@ else git checkout ${GIT_BRANCH} git pull git checkout ${GIT_COMMIT} + git fetch --tags + git submodule update --init --recursive fi export MORPHEUS_ROOT=$(pwd) diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh index 3bfa4e0870..75e83a0c7c 100644 --- a/ci/scripts/common.sh +++ b/ci/scripts/common.sh @@ -73,7 +73,7 @@ function get_modified_files() { local GIT_DIFF_BASE=${GIT_DIFF_BASE:-$(get_merge_base)} # If invoked by a git-commit-hook, this will be populated - local result=( $(git diff ${GIT_DIFF_ARGS} $(get_merge_base) | grep -P ${1:-'.*'}) ) + local result=( $(git diff ${GIT_DIFF_ARGS} ${GIT_DIFF_BASE} | grep -P ${1:-'.*'}) ) local files=() diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh index b75107f637..71870b1135 100755 --- a/ci/scripts/github/build.sh +++ b/ci/scripts/github/build.sh @@ -20,42 +20,34 @@ source ${WORKSPACE}/ci/scripts/github/common.sh rapids-dependency-file-generator \ --output conda \ - --file_key build \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --file-key build \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml" -update_conda_env env.yaml +update_conda_env "${WORKSPACE_TMP}/env.yaml" log_toolchain -git submodule update --init --recursive - CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}" CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_WHEEL=ON" CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF" CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON" -if [[ "${LOCAL_CI}" == "" ]]; then - CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)" -fi rapids-logger "Configuring cmake for Morpheus with ${CMAKE_FLAGS}" -cmake -B build -G Ninja ${CMAKE_FLAGS} . +cmake ${CMAKE_FLAGS} . rapids-logger "Building Morpheus" -cmake --build build --parallel ${PARALLEL_LEVEL} +cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL} -if [[ "${LOCAL_CI}" == "" ]]; then - rapids-logger "sccache usage for morpheus build:" - sccache --show-stats -fi +log_sccache_stats rapids-logger "Archiving results" -tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" build/dist +tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" ${BUILD_DIR}/dist -MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/build/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \ +MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \ $(find ${MORPHEUS_ROOT}/examples -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;)) tar cfj "${WORKSPACE_TMP}/morhpeus_libs.tar.bz" "${MORPHEUS_LIBS[@]}" -CPP_TESTS=($(find ${MORPHEUS_ROOT}/build/morpheus/_lib/tests -name "*.x" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;)) +CPP_TESTS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/morpheus/_lib/tests -name "*.x" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;)) tar cfj "${WORKSPACE_TMP}/cpp_tests.tar.bz" "${CPP_TESTS[@]}" rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}" diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh index 487e053c7a..5f90a50828 100755 --- a/ci/scripts/github/checks.sh +++ b/ci/scripts/github/checks.sh @@ -20,43 +20,35 @@ source ${WORKSPACE}/ci/scripts/github/common.sh rapids-dependency-file-generator \ --output conda \ - --file_key build \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --file-key build \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml" -update_conda_env env.yaml +update_conda_env "${WORKSPACE_TMP}/env.yaml" log_toolchain cd ${MORPHEUS_ROOT} +# Fetching the base branch will try methods that might fail, then fallback to one that does, set +e for this section +set +e fetch_base_branch - -git submodule update --init --recursive +set -e rapids-logger "Configuring cmake for Morpheus" CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}" CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF" export CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_INPLACE_BUILD=ON" -if [[ "${LOCAL_CI}" == "" ]]; then - CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)" -fi -cmake -B build -G Ninja ${CMAKE_FLAGS} . +cmake ${CMAKE_FLAGS} . rapids-logger "Building Morpheus" -cmake --build build --parallel ${PARALLEL_LEVEL} +cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL} -if [[ "${LOCAL_CI}" == "" ]]; then - rapids-logger "sccache usage for source build:" - sccache --show-stats -fi +log_sccache_stats rapids-logger "Installing Morpheus" pip install ./ -# Setting this prevents loading of cudf since we don't have a GPU -export MORPHEUS_IN_SPHINX_BUILD=1 - rapids-logger "Checking copyright headers" python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT} diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index 3aa6c4c69e..5f50779726 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -61,7 +61,30 @@ export SCCACHE_REGION="us-east-2" export SCCACHE_IDLE_TIMEOUT=32768 #export SCCACHE_LOG=debug -export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_PYTHON_BUILD_STUBS=ON -DMORPHEUS_USE_CCACHE=ON" +# Set the build flags +export BUILD_DIR=${BUILD_DIR:-build} + +_FLAGS=() +_FLAGS+=("-B" "${BUILD_DIR}") +_FLAGS+=("-G" "Ninja") +_FLAGS+=("-DCMAKE_MESSAGE_CONTEXT_SHOW=ON") +_FLAGS+=("-DMORPHEUS_CUDA_ARCHITECTURES=RAPIDS") +_FLAGS+=("-DMORPHEUS_USE_CONDA=ON") +_FLAGS+=("-DMORPHEUS_USE_CCACHE=ON") +_FLAGS+=("-DMORPHEUS_PYTHON_INPLACE_BUILD=OFF") +_FLAGS+=("-DMORPHEUS_PYTHON_BUILD_STUBS=ON") +_FLAGS+=("-DMORPHEUS_BUILD_BENCHMARKS=ON") +_FLAGS+=("-DMORPHEUS_BUILD_EXAMPLES=ON") +_FLAGS+=("-DMORPHEUS_BUILD_TESTS=ON") +if [[ "${LOCAL_CI}" == "" ]]; then + _FLAGS+=("-DCCACHE_PROGRAM_PATH=$(which sccache)") +fi +export CMAKE_BUILD_ALL_FEATURES="${_FLAGS[@]}" +unset _FLAGS + +if [[ ${MORPHEUS_SUPPORT_DOCA} == @(TRUE|ON) ]]; then + export CMAKE_BUILD_ALL_FEATURES="${CMAKE_BUILD_ALL_FEATURES} -DMORPHEUS_SUPPORT_DOCA=ON" +fi export FETCH_STATUS=0 @@ -112,8 +135,11 @@ function fetch_base_branch_gh_api() { function fetch_base_branch_local() { rapids-logger "Retrieving base branch from git" - git remote add upstream ${GIT_UPSTREAM_URL} - git fetch upstream --tags + if [[ "${USE_HOST_GIT}" == "0" ]]; then + git remote add upstream ${GIT_UPSTREAM_URL} + git fetch upstream --tags + fi + source ${MORPHEUS_ROOT}/ci/scripts/common.sh export BASE_BRANCH=$(get_base_branch) export CHANGE_TARGET="upstream/${BASE_BRANCH}" @@ -147,6 +173,13 @@ function log_toolchain() { sccache --version } +function log_sccache_stats() { + if [[ "${LOCAL_CI}" == "" ]]; then + rapids-logger "sccache usage for morpheus build:" + sccache --show-stats + fi +} + function upload_artifact() { FILE_NAME=$1 BASE_NAME=$(basename "${FILE_NAME}") diff --git a/ci/scripts/github/conda.sh b/ci/scripts/github/conda.sh index f92374f222..4114bd9ab9 100755 --- a/ci/scripts/github/conda.sh +++ b/ci/scripts/github/conda.sh @@ -23,8 +23,6 @@ cd ${MORPHEUS_ROOT} fetch_base_branch -git submodule update --init --recursive - # Its important that we are in the base environment for the build rapids-logger "Activating Base Conda Environment" diff --git a/ci/scripts/github/docs.sh b/ci/scripts/github/docs.sh index f928d02a38..0a5696188d 100755 --- a/ci/scripts/github/docs.sh +++ b/ci/scripts/github/docs.sh @@ -20,16 +20,16 @@ source ${WORKSPACE}/ci/scripts/github/common.sh rapids-dependency-file-generator \ --output conda \ - --file_key docs \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --file-key docs \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml" -update_conda_env env.yaml +update_conda_env "${WORKSPACE_TMP}/env.yaml" download_artifact "wheel.tar.bz" tar xf "${WORKSPACE_TMP}/wheel.tar.bz" -pip install ${MORPHEUS_ROOT}/build/dist/*.whl +pip install ${MORPHEUS_ROOT}/${BUILD_DIR}/dist/*.whl rapids-logger "Pulling LFS assets" cd ${MORPHEUS_ROOT} @@ -37,17 +37,15 @@ cd ${MORPHEUS_ROOT} git lfs install ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch docs examples -git submodule update --init --recursive - rapids-logger "Configuring for docs" -cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF -DMORPHEUS_BUILD_DOCS=ON . +cmake ${CMAKE_BUILD_ALL_FEATURES} -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF -DMORPHEUS_BUILD_DOCS=ON . rapids-logger "Building docs" -cmake --build build --parallel ${PARALLEL_LEVEL} --target install -cmake --build build --parallel ${PARALLEL_LEVEL} --target morpheus_docs +cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL} --target install +cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL} --target morpheus_docs rapids-logger "Archiving the docs" -tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/docs/html +tar cfj "${WORKSPACE_TMP}/docs.tar.bz" ${BUILD_DIR}/docs/html rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}" set_job_summary_preamble diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh index fe4fe23813..38aabd5cd5 100755 --- a/ci/scripts/github/test.sh +++ b/ci/scripts/github/test.sh @@ -21,35 +21,27 @@ source ${WORKSPACE}/ci/scripts/github/common.sh rapids-dependency-file-generator \ --output conda \ - --file_key test \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --file-key all \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml" -update_conda_env env.yaml +update_conda_env "${WORKSPACE_TMP}/env.yaml" log_toolchain -git submodule update --init --recursive - CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}" CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON" CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=ON" CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_WHEEL=OFF" CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_PERFORM_INSTALL=ON" CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX}" -if [[ "${LOCAL_CI}" == "" ]]; then - CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)" -fi rapids-logger "Configuring cmake for Morpheus with ${CMAKE_FLAGS}" -cmake -B build -G Ninja ${CMAKE_FLAGS} . +cmake ${CMAKE_FLAGS} . rapids-logger "Building Morpheus" -cmake --build build --parallel ${PARALLEL_LEVEL} --target install +cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL} --target install -if [[ "${LOCAL_CI}" == "" ]]; then - rapids-logger "sccache usage for morpheus build:" - sccache --show-stats -fi +log_sccache_stats rapids-logger "Checking Python stub files" @@ -62,7 +54,7 @@ if [[ $(git status --short --untracked | grep .pyi) != "" ]]; then exit 1 fi -CPP_TESTS=($(find ${MORPHEUS_ROOT}/build -name "*.x")) +CPP_TESTS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR} -name "*.x")) rapids-logger "Pulling LFS assets" diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh index fb29fdf139..1b6816f5d1 100755 --- a/ci/scripts/run_ci_local.sh +++ b/ci/scripts/run_ci_local.sh @@ -45,7 +45,10 @@ MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)} # match CI, the default) USE_HOST_GIT=${USE_HOST_GIT:-0} -GIT_URL=$(git remote get-url origin) +# Useful when using a host git repo to avoid conflicting with a potentially existing 'build' directory +BUILD_DIR=${BUILD_DIR:-build-ci} + +GIT_URL=${GIT_URL:-$(git remote get-url origin)} GIT_URL=$(git_ssh_to_https ${GIT_URL}) GIT_UPSTREAM_URL=$(git remote get-url upstream) @@ -55,40 +58,48 @@ GIT_BRANCH=$(git branch --show-current) GIT_COMMIT=$(git log -n 1 --pretty=format:%H) LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp} -CONTAINER_VER=${CONTAINER_VER:-240221} +CONTAINER_VER=${CONTAINER_VER:-240614} CUDA_VER=${CUDA_VER:-12.1} DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-${CONTAINER_VER}" TEST_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-${CONTAINER_VER}" -ENV_LIST="--env LOCAL_CI_TMP=/ci_tmp" -ENV_LIST="${ENV_LIST} --env GIT_URL=${GIT_URL}" -ENV_LIST="${ENV_LIST} --env GIT_UPSTREAM_URL=${GIT_UPSTREAM_URL}" -ENV_LIST="${ENV_LIST} --env GIT_BRANCH=${GIT_BRANCH}" -ENV_LIST="${ENV_LIST} --env GIT_COMMIT=${GIT_COMMIT}" -ENV_LIST="${ENV_LIST} --env PARALLEL_LEVEL=$(nproc)" -ENV_LIST="${ENV_LIST} --env CUDA_VER=${CUDA_VER}" -ENV_LIST="${ENV_LIST} --env SKIP_CONDA_ENV_UPDATE=${SKIP_CONDA_ENV_UPDATE}" -ENV_LIST="${ENV_LIST} --env USE_HOST_GIT=${USE_HOST_GIT}" +ENV_LIST=() +ENV_LIST+=("--env" "LOCAL_CI_TMP=/ci_tmp") +ENV_LIST+=("--env" "GIT_URL=${GIT_URL}") +ENV_LIST+=("--env" "GIT_UPSTREAM_URL=${GIT_UPSTREAM_URL}") +ENV_LIST+=("--env" "GIT_BRANCH=${GIT_BRANCH}") +ENV_LIST+=("--env" "GIT_COMMIT=${GIT_COMMIT}") +ENV_LIST+=("--env" "PARALLEL_LEVEL=$(nproc)") +ENV_LIST+=("--env" "CUDA_VER=${CUDA_VER}") +ENV_LIST+=("--env" "SKIP_CONDA_ENV_UPDATE=${SKIP_CONDA_ENV_UPDATE}") +ENV_LIST+=("--env" "USE_HOST_GIT=${USE_HOST_GIT}") +ENV_LIST+=("--env" "BUILD_DIR=${BUILD_DIR}") mkdir -p ${LOCAL_CI_TMP} cp ${MORPHEUS_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP} for STAGE in "${STAGES[@]}"; do - DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}" + DOCKER_RUN_ARGS=() + DOCKER_RUN_ARGS+=("--rm") + DOCKER_RUN_ARGS+=("-ti") + DOCKER_RUN_ARGS+=("--net=host") + DOCKER_RUN_ARGS+=("-v" "${LOCAL_CI_TMP}:/ci_tmp") + DOCKER_RUN_ARGS+=("${ENV_LIST[@]}") + DOCKER_RUN_ARGS+=("--env STAGE=${STAGE}") if [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then CONTAINER="${TEST_CONTAINER}" - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia" - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --gpus all" - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --cap-add=sys_nice" + DOCKER_RUN_ARGS+=("--runtime=nvidia") + DOCKER_RUN_ARGS+=("--gpus all") + DOCKER_RUN_ARGS+=("--cap-add=sys_nice") else CONTAINER="${BUILD_CONTAINER}" - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc" + DOCKER_RUN_ARGS+=("--runtime=runc") fi if [[ "${USE_HOST_GIT}" == "1" ]]; then - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} -v ${MORPHEUS_ROOT}:/Morpheus" + DOCKER_RUN_ARGS+=("-v" "${MORPHEUS_ROOT}:/Morpheus") fi if [[ "${STAGE}" == "bash" ]]; then @@ -99,7 +110,7 @@ for STAGE in "${STAGES[@]}"; do echo "Running ${STAGE} stage in ${CONTAINER}" set -x - docker run ${DOCKER_RUN_ARGS} ${DOCKER_EXTRA_ARGS} ${CONTAINER} ${DOCKER_RUN_CMD} + docker run ${DOCKER_RUN_ARGS[@]} ${DOCKER_EXTRA_ARGS} ${CONTAINER} ${DOCKER_RUN_CMD} set +x STATUS=$? diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml index 66dc9a2f60..f860ece4c6 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-121_arch-x86_64.yaml @@ -12,39 +12,42 @@ dependencies: - anyio>=3.7 - appdirs - arxiv=1.4 -- automake +- automake=1.16.5 +- beautifulsoup4=4.12 - benchmark=1.8.3 -- boost-cpp=1.84 - boto3 - breathe=4.35.0 - ccache - clangdev=16 -- click >=8 +- click>=8 - cmake=3.27 - cuda-cudart-dev=12.1 -- cuda-nvcc +- cuda-cudart=12.1 +- cuda-nvcc=12.1 - cuda-nvml-dev=12.1 - cuda-nvrtc-dev=12.1 +- cuda-nvrtc=12.1 - cuda-nvtx-dev=12.1 +- cuda-nvtx=12.1 - cuda-tools=12.1 - cuda-version=12.1 - cudf=24.02 - cuml=24.02.* +- cupy - cxx-compiler - cython=3.0 -- dask=2023.12.1 - datacompy=0.10 - dill=0.3.7 -- distributed=2023.12.1 +- docker-py=5.0 - doxygen=1.9.2 - elasticsearch==8.9.0 - exhale=0.3.6 -- feedparser=6.0.10 +- feedparser=6.0 - flake8 - gcc_linux-64=11.2 - git-lfs - glog=0.6 -- grpcio-status==1.59 +- grpcio-status=1.59 - grpcio=1.59 - gxx_linux-64=11.2 - huggingface_hub=0.20.2 @@ -53,11 +56,12 @@ dependencies: - isort - jsonpatch>=1.33 - kfp +- libcudf=24.02 - librdkafka>=1.9.2,<1.10.0a0 - libtool - libwebp=1.3.2 - mlflow>=2.10.0,<3 -- mrc=24.03 +- mrc=24.06 - myst-parser=0.18.1 - nbsphinx - networkx=2.8.8 @@ -68,14 +72,13 @@ dependencies: - numexpr - numpydoc=1.5 - nvtabular=23.08.00 -- onnx +- onnx=1.15 - openai=1.13 - papermill=2.4.0 - pip - pkg-config=0.29 - pluggy=1.3 - pre-commit -- protobuf=4.24 - pybind11-stubgen=0.10.5 - pydantic - pylint=3.0.3 @@ -91,24 +94,25 @@ dependencies: - pytorch-cuda - pytorch=*=*cuda* - rapidjson=1.1.0 +- rapids-dask-dependency=24.02 - rdma-core>=48 - requests - requests-cache=1.1 -- requests-toolbelt +- requests-toolbelt=1.0 - s3fs=2023.12.2 - scikit-build=0.17.6 - scikit-learn=1.3.2 -- sentence-transformers +- sentence-transformers=2.7 - sphinx - sphinx_rtd_theme -- sqlalchemy +- sqlalchemy<2.0 - sysroot_linux-64=2.17 - tqdm=4 - transformers=4.36.2 - tritonclient=2.34 - typing_utils=0.1 -- ucx=1.15 - versioneer +- versioneer-518 - watchdog=3.0 - websockets - yapf=0.40.1 @@ -116,15 +120,17 @@ dependencies: - pip: - --find-links https://data.dgl.ai/wheels-test/repo.html - --find-links https://data.dgl.ai/wheels/cu121/repo.html - - PyMuPDF==1.23.21 + - PyMuPDF==1.23.* - databricks-cli < 0.100 - databricks-connect - dgl==2.0.0 - dglgo + - faiss-gpu==1.7.* - google-search-results==2.4 - - langchain==0.1.9 + - langchain-nvidia-ai-endpoints==0.0.11 + - langchain==0.1.16 - milvus==2.3.5 - - nemollm + - nemollm==0.3.5 - pymilvus==2.3.6 - pytest-kafka==0.6.0 name: all_cuda-121_arch-x86_64 diff --git a/conda/environments/dev_cuda-121_arch-x86_64.yaml b/conda/environments/dev_cuda-121_arch-x86_64.yaml index 24c87ba34f..30781144a4 100644 --- a/conda/environments/dev_cuda-121_arch-x86_64.yaml +++ b/conda/environments/dev_cuda-121_arch-x86_64.yaml @@ -10,43 +10,48 @@ channels: - pytorch dependencies: - appdirs -- automake +- automake=1.16.5 +- beautifulsoup4=4.12 - benchmark=1.8.3 -- boost-cpp=1.84 - breathe=4.35.0 - ccache - clangdev=16 -- click >=8 +- click>=8 - cmake=3.27 - cuda-cudart-dev=12.1 -- cuda-nvcc +- cuda-nvcc=12.1 - cuda-nvml-dev=12.1 - cuda-nvrtc-dev=12.1 - cuda-nvtx-dev=12.1 - cuda-tools=12.1 - cuda-version=12.1 - cudf=24.02 +- cupy - cxx-compiler - cython=3.0 - datacompy=0.10 - dill=0.3.7 +- docker-py=5.0 - doxygen=1.9.2 - elasticsearch==8.9.0 - exhale=0.3.6 -- feedparser=6.0.10 +- feedparser=6.0 - flake8 - gcc_linux-64=11.2 - git-lfs - glog=0.6 +- grpcio-status=1.59 - grpcio=1.59 - gxx_linux-64=11.2 - include-what-you-use=0.20 - ipython - isort +- libcudf=24.02 - librdkafka>=1.9.2,<1.10.0a0 - libtool +- libwebp=1.3.2 - mlflow>=2.10.0,<3 -- mrc=24.03 +- mrc=24.06 - myst-parser=0.18.1 - nbsphinx - networkx=2.8.8 @@ -59,7 +64,6 @@ dependencies: - pkg-config=0.29 - pluggy=1.3 - pre-commit -- protobuf=4.24 - pybind11-stubgen=0.10.5 - pydantic - pylint=3.0.3 @@ -74,27 +78,27 @@ dependencies: - pytorch-cuda - pytorch=*=*cuda* - rapidjson=1.1.0 +- rapids-dask-dependency=24.02 - rdma-core>=48 - requests - requests-cache=1.1 -- requests-toolbelt - scikit-build=0.17.6 - scikit-learn=1.3.2 - sphinx - sphinx_rtd_theme -- sqlalchemy +- sqlalchemy<2.0 - sysroot_linux-64=2.17 - tqdm=4 - tritonclient=2.34 - typing_utils=0.1 -- ucx=1.15 - versioneer +- versioneer-518 - watchdog=3.0 - websockets - yapf=0.40.1 - zlib=1.2.13 - pip: - - PyMuPDF==1.23.21 + - PyMuPDF==1.23.* - databricks-cli < 0.100 - databricks-connect - milvus==2.3.5 diff --git a/conda/environments/examples_cuda-121_arch-x86_64.yaml b/conda/environments/examples_cuda-121_arch-x86_64.yaml index 857b73aa85..b144b38b13 100644 --- a/conda/environments/examples_cuda-121_arch-x86_64.yaml +++ b/conda/environments/examples_cuda-121_arch-x86_64.yaml @@ -12,62 +12,71 @@ dependencies: - anyio>=3.7 - appdirs - arxiv=1.4 +- beautifulsoup4=4.12 - boto3 -- click >=8 +- click>=8 +- cudf=24.02 - cuml=24.02.* -- dask=2023.12.1 +- cupy - datacompy=0.10 - dill=0.3.7 -- distributed=2023.12.1 +- docker-py=5.0 - elasticsearch==8.9.0 -- feedparser=6.0.10 -- grpcio-status==1.59 +- feedparser=6.0 +- grpcio-status=1.59 - grpcio=1.59 - huggingface_hub=0.20.2 - jsonpatch>=1.33 - kfp - libwebp=1.3.2 - mlflow>=2.10.0,<3 +- mrc=24.06 - networkx=2.8.8 - newspaper3k=0.2 - nodejs=18.* - numexpr - numpydoc=1.5 - nvtabular=23.08.00 -- onnx +- onnx=1.15 - openai=1.13 - papermill=2.4.0 - pip +- pluggy=1.3 - pydantic - pypdf=3.17.4 - python-confluent-kafka>=1.9.2,<1.10.0a0 +- python-docx==1.1.0 - python-graphviz - python=3.10 - pytorch-cuda - pytorch=*=*cuda* +- rapids-dask-dependency=24.02 - requests - requests-cache=1.1 -- requests-toolbelt +- requests-toolbelt=1.0 - s3fs=2023.12.2 - scikit-learn=1.3.2 -- sentence-transformers -- sqlalchemy +- sentence-transformers=2.7 +- sqlalchemy<2.0 - tqdm=4 - transformers=4.36.2 +- tritonclient=2.34 - typing_utils=0.1 - watchdog=3.0 - websockets - pip: - --find-links https://data.dgl.ai/wheels-test/repo.html - --find-links https://data.dgl.ai/wheels/cu121/repo.html - - PyMuPDF==1.23.21 + - PyMuPDF==1.23.* - databricks-cli < 0.100 - databricks-connect - dgl==2.0.0 - dglgo + - faiss-gpu==1.7.* - google-search-results==2.4 - - langchain==0.1.9 + - langchain-nvidia-ai-endpoints==0.0.11 + - langchain==0.1.16 - milvus==2.3.5 - - nemollm + - nemollm==0.3.5 - pymilvus==2.3.6 name: examples_cuda-121_arch-x86_64 diff --git a/conda/environments/runtime_cuda-121_arch-x86_64.yaml b/conda/environments/runtime_cuda-121_arch-x86_64.yaml index 3f9543d426..b7ab668e1f 100644 --- a/conda/environments/runtime_cuda-121_arch-x86_64.yaml +++ b/conda/environments/runtime_cuda-121_arch-x86_64.yaml @@ -10,29 +10,42 @@ channels: - pytorch dependencies: - appdirs -- click >=8 +- beautifulsoup4=4.12 +- click>=8 +- cuda-cudart=12.1 +- cuda-nvrtc=12.1 +- cuda-nvtx=12.1 +- cuda-version=12.1 +- cudf=24.02 +- cupy - datacompy=0.10 - dill=0.3.7 +- docker-py=5.0 - elasticsearch==8.9.0 -- feedparser=6.0.10 +- feedparser=6.0 +- grpcio-status=1.59 - grpcio=1.59 +- libwebp=1.3.2 - mlflow>=2.10.0,<3 +- mrc=24.06 - networkx=2.8.8 - numpydoc=1.5 - nvtabular=23.08.00 - pip +- pluggy=1.3 - pydantic - python-confluent-kafka>=1.9.2,<1.10.0a0 - python-graphviz - python=3.10 - pytorch-cuda - pytorch=*=*cuda* +- rapids-dask-dependency=24.02 - requests - requests-cache=1.1 -- requests-toolbelt - scikit-learn=1.3.2 -- sqlalchemy +- sqlalchemy<2.0 - tqdm=4 +- tritonclient=2.34 - typing_utils=0.1 - watchdog=3.0 - websockets diff --git a/dependencies.yaml b/dependencies.yaml index 64b03c5f3f..089e0d24c9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -15,6 +15,7 @@ # Dependency list for https://github.com/rapidsai/dependency-file-generator files: + # Includes all dependencies together in a single file all: output: conda matrix: @@ -22,24 +23,25 @@ files: arch: [x86_64] includes: - benchmark_cpp - - build_cpp + - build_cpp_build + - build_cpp_host - checks - cudatoolkit + - cudatoolkit-dev - cve-mitigation - data_retrieval - development - - doca - docs - example-dfp-prod - example-gnn - - example-llm-agents - - example-llm-completion - - example-llm-rag - - example-llm-vdb-upload + - example-llms - python - runtime - test_python_morpheus + # Common dependencies for developing with Morpheus. + # Includes: runtime, build, docs, test + # Excludes: examples dev: output: conda matrix: @@ -47,17 +49,21 @@ files: arch: [x86_64] includes: - benchmark_cpp - - build_cpp + - build_cpp_build + - build_cpp_host - checks - - cudatoolkit + - cudatoolkit-dev + - cve-mitigation - data_retrieval - development - - doca - docs - python - runtime - test_python_morpheus + # Dependencies that are needed to build Morpheus. + # Includes: runtime + # Excludes: test, examples build: output: none matrix: @@ -65,14 +71,19 @@ files: arch: [x86_64] includes: - benchmark_cpp - - build_cpp - - cudatoolkit + - build_cpp_build + - build_cpp_host + - cudatoolkit-dev + - cve-mitigation - data_retrieval - development - python - runtime - test_python_morpheus + # Dependencies that are needed to run Morpheus tests. + # Includes: runtime, examples + # Excludes: build, dev test: output: none matrix: @@ -80,20 +91,20 @@ files: arch: [x86_64] includes: - benchmark_cpp - - build_cpp - cudatoolkit + - cve-mitigation - data_retrieval - development - example-dfp-prod - example-gnn - - example-llm-agents - - example-llm-completion - - example-llm-rag - - example-llm-vdb-upload + - example-llms - python - runtime - test_python_morpheus + # Dependencies that are needed to generate the Morpheus documentation. + # Includes: build, runtime, examples + # Excludes: test docs: output: none matrix: @@ -101,28 +112,36 @@ files: arch: [x86_64] includes: - benchmark_cpp - - build_cpp - - cudatoolkit + - build_cpp_build + - build_cpp_host + - cudatoolkit-dev + - cve-mitigation - data_retrieval - docs - example-dfp-prod - example-gnn - - example-llm-agents - - example-llm-completion - - example-llm-rag - - example-llm-vdb-upload + - example-llms - python - runtime + # Only the dependencies that are needed to run core Morpheus. This is the smallest set of dependencies needed to run + # a Morpheus pipeline. + # Includes: none + # Excludes: build, dev, test, examples, docs runtime: output: conda matrix: cuda: ["12.1"] arch: [x86_64] includes: - - runtime + - cudatoolkit + - cve-mitigation - python + - runtime + # Dependencies that are needed to run all Morpheus examples. + # Includes: runtime + # Excludes: build, dev, test examples: output: conda matrix: @@ -132,13 +151,13 @@ files: - cve-mitigation - example-dfp-prod - example-gnn - - example-llm-agents - - example-llm-completion - - example-llm-rag - - example-llm-vdb-upload + - example-llms - python - runtime + # Dependencies which are needed to run the model generation scripts in the `models` directory. + # Includes: none + # Excludes: runtime, build, dev, test, examples, docs model-utils: output: conda matrix: @@ -148,6 +167,7 @@ files: - model-training-tuning - python + # Dependencies which are used during the CI "checks" stage. Does not output any files. checks: output: none includes: @@ -165,32 +185,82 @@ channels: dependencies: - build_cpp: # should be split into `build_python` if/when converting to use scikit-build + python: + common: + - output_types: [conda] + packages: + - python=3.10 + + cudatoolkit: + specific: + - output_types: [conda] + matrices: + - matrix: + cuda: "12.1" + packages: + - cuda-cudart=12.1 + - cuda-nvrtc=12.1 + - cuda-nvtx=12.1 + - cuda-version=12.1 + + cudatoolkit-dev: + specific: + - output_types: [conda] + matrices: + - matrix: + cuda: "12.1" + packages: + - cuda-cudart-dev=12.1 + - cuda-nvml-dev=12.1 + - cuda-nvrtc-dev=12.1 + - cuda-nvtx-dev=12.1 + - cuda-tools=12.1 + - cuda-version=12.1 + + + + # Build dependencies for Morpheus on the target arch. Mirrors the `build` section in + # ci/conda/recipes/morpheus/meta.yaml + build_cpp_build: common: - output_types: [conda] packages: - - boost-cpp=1.84 + # Compilers + - cuda-nvcc=12.1 + - cxx-compiler + - gcc_linux-64=11.2 + - gxx_linux-64=11.2 + + # Non-Compiler Dependencies + - automake=1.16.5 # Needed for DOCA build - ccache - cmake=3.27 - - cuda-nvcc + - cuda-cudart-dev=12.1 + - cuda-version=12.1 + - libtool # Needed for DOCA build + - ninja=1.11 + - pkg-config=0.29 # for mrc cmake + - sysroot_linux-64=2.17 + + # Build dependencies for Morpheus on the host arch. Mirrors the `host` section in + # ci/conda/recipes/morpheus/meta.yaml + build_cpp_host: + common: + - output_types: [conda] + packages: + # Include: cudatoolkit-dev - cudf=24.02 - - cxx-compiler - cython=3.0 - - gcc_linux-64=11.2 - glog=0.6 - - gxx_linux-64=11.2 + - libcudf=24.02 - librdkafka>=1.9.2,<1.10.0a0 - - mrc=24.03 - - ninja=1.11 + - mrc=24.06 - nlohmann_json=3.11 - - pkg-config=0.29 # for mrc cmake - - protobuf=4.24 - pybind11-stubgen=0.10.5 - rapidjson=1.1.0 + - rdma-core>=48 # Needed for DOCA. - scikit-build=0.17.6 - - sysroot_linux-64=2.17 - - tritonclient=2.34 - - ucx=1.15 + - versioneer-518 - zlib=1.2.13 checks: @@ -221,16 +291,12 @@ dependencies: common: - output_types: [conda] packages: - - &click click >=8 - - &numpydoc numpydoc=1.5 - breathe=4.35.0 - doxygen=1.9.2 - exhale=0.3.6 - ipython - myst-parser=0.18.1 - nbsphinx - - pluggy=1.3 - - python-graphviz - sphinx - sphinx_rtd_theme @@ -240,32 +306,49 @@ dependencies: packages: - benchmark=1.8.3 + # Runtime dependencies for Morpheus. Included in nearly all output files so dependencies should + # be added only if it is needed to run the core Morpheus library. + # This should be synced with `run` in ci/conda/recipes/morpheus/meta.yaml runtime: common: - output_types: [conda] packages: - - *click - - *numpydoc - - &dill dill=0.3.7 - - &scikit-learn scikit-learn=1.3.2 + # Include: cudatoolkit + # Include: python + # Include: cve-mitigation - appdirs + - beautifulsoup4=4.12 + - click>=8 + # - cuda-version=12.1 ## + - cudf=24.02 + - cupy # Version determined from cudf - datacompy=0.10 + - dill=0.3.7 + - docker-py=5.0 - elasticsearch==8.9.0 - - feedparser=6.0.10 + - feedparser=6.0 - grpcio=1.59 + - grpcio-status=1.59 + # - libwebp=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863 ## - mlflow>=2.10.0,<3 + - mrc=24.06 - networkx=2.8.8 + - numpydoc=1.5 - nvtabular=23.08.00 - pydantic + # - python ## - python-confluent-kafka>=1.9.2,<1.10.0a0 - python-graphviz - pytorch-cuda - pytorch=*=*cuda* + - pluggy=1.3 + - rapids-dask-dependency=24.02 # provides dask and distributed - requests - requests-cache=1.1 - - requests-toolbelt # Transitive dep needed by nemollm, specified here to ensure we get a compatible version - - sqlalchemy + - scikit-learn=1.3.2 + - sqlalchemy<2.0 # 2.0 is incompatible with pandas=1.3 - tqdm=4 + - tritonclient=2.34 - typing_utils=0.1 - watchdog=3.0 - websockets @@ -285,23 +368,19 @@ dependencies: - pytest-benchmark=4.0 - pytest-cov - pytest=7.4.4 - - python-docx==1.1.0 + - &python-docx python-docx==1.1.0 - pip - pip: - - PyMuPDF==1.23.21 + - &PyMuPDF PyMuPDF==1.23.* - pytest-kafka==0.6.0 example-dfp-prod: common: - output_types: [conda] packages: - - *dill - *nodejs - boto3 - - dask=2023.12.1 - - distributed=2023.12.1 - kfp - - mlflow>=2.10.0,<3 - papermill=2.4.0 - s3fs=2023.12.2 @@ -317,63 +396,39 @@ dependencies: - dgl==2.0.0 - dglgo - example-llm-agents: + example-llms: common: - output_types: [conda] packages: - - &grpcio-status grpcio-status==1.59 - &transformers transformers=4.36.2 # newer versions are incompatible with our pinned version of huggingface_hub - - huggingface_hub=0.20.2 # work-around for https://github.com/UKPLab/sentence-transformers/issues/1762 - - numexpr - - sentence-transformers - - pip - - pip: - - &langchain langchain==0.1.9 - - nemollm - - example-llm-completion: - common: - - output_types: [conda] - packages: - - *grpcio-status - - &arxiv arxiv=1.4 - - &newspaper3k newspaper3k=0.2 - - &pypdf pypdf=3.17.4 - - example-llm-rag: - common: - - output_types: [conda] - packages: - - *grpcio-status - anyio>=3.7 + - arxiv=1.4 + - huggingface_hub=0.20.2 # work-around for https://github.com/UKPLab/sentence-transformers/issues/1762 - jsonpatch>=1.33 + - newspaper3k=0.2 + - numexpr + - onnx=1.15 - openai=1.13 + - pypdf=3.17.4 + - *python-docx + - requests-toolbelt=1.0 # Transitive dep needed by nemollm, specified here to ensure we get a compatible version + - sentence-transformers=2.7 - pip - pip: - - *langchain + - langchain==0.1.16 + - langchain-nvidia-ai-endpoints==0.0.11 + - faiss-gpu==1.7.* - google-search-results==2.4 - - example-llm-vdb-upload: - common: - - output_types: [conda] - packages: - - *arxiv - - *grpcio-status - - *newspaper3k - - *pypdf - - onnx - - pip - - pip: - - PyMuPDF==1.23.21 - - *langchain + - nemollm==0.3.5 + - *PyMuPDF model-training-tuning: common: - output_types: [conda] packages: - *cuml - - *scikit-learn - *transformers + - scikit-learn=1.3.2 - jupyterlab - matplotlib - onnx @@ -387,32 +442,3 @@ dependencies: - output_types: [conda] packages: - libwebp=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863 - - - doca: - common: - - output_types: [conda] - packages: - - automake - - libtool - - rdma-core>=48 - - python: - common: - - output_types: [conda] - packages: - - python=3.10 - - cudatoolkit: - specific: - - output_types: [conda] - matrices: - - matrix: - cuda: "12.1" - packages: - - cuda-cudart-dev=12.1 - - cuda-nvml-dev=12.1 - - cuda-nvrtc-dev=12.1 - - cuda-nvtx-dev=12.1 - - cuda-tools=12.1 - - cuda-version=12.1 diff --git a/docker/Dockerfile b/docker/Dockerfile index 492e4becae..6f67668571 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,7 @@ # ============== LEGEND =================================================== # # [conda_bld_morpheus] # / -# [base] -> [conda_env] -> [base_extended] -> [runtime] +# [base] -> [conda_env] -> [base_extended] -> [runtime_conda_create] -> [runtime] # \ # [conda_env_dev] -> [development] -> [development_pydbg] # @@ -153,7 +153,7 @@ ARG MORPHEUS_SUPPORT_DOCA="FALSE" ENV MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA} # Copy all of the optional dependency scripts -COPY ${MORPHEUS_ROOT_HOST}/docker/optional_deps docker/optional_deps +COPY ${MORPHEUS_ROOT_HOST}/.devcontainer/docker/optional_deps docker/optional_deps # Install DOCA (If requested) RUN --mount=type=cache,id=doca,target=/tmp/doca,sharing=locked \ @@ -188,6 +188,8 @@ ARG MORPHEUS_ROOT_HOST ARG CUDA_MAJOR_VER ARG CUDA_MINOR_VER +ENV GIT_CLONE_PROTECTION_ACTIVE=false + RUN --mount=type=cache,id=apt,target=/var/cache/apt \ apt update && \ DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \ @@ -218,23 +220,14 @@ RUN --mount=type=cache,id=workspace_cache,target=/workspace/.cache,sharing=locke cd ${MORPHEUS_ROOT_HOST} &&\ MORPHEUS_PYTHON_BUILD_STUBS=OFF CONDA_BLD_PATH=/opt/conda/conda-bld ./ci/conda/recipes/run_conda_build.sh morpheus -# ============ Stage: runtime ============ -# Setup container for runtime environment -FROM base_extended as runtime - +# ============ Stage: runtime_conda_create ============ +# Setup conda for the runtime environment +FROM base_extended as runtime_conda_create ARG MORPHEUS_ROOT_HOST ARG CUDA_MAJOR_VER ARG CUDA_MINOR_VER -# Only copy specific files/folders over that are necessary for runtime -COPY "${MORPHEUS_ROOT_HOST}/conda/environments/*.yaml" "./conda/environments/" -COPY "${MORPHEUS_ROOT_HOST}/docker" "./docker" -COPY "${MORPHEUS_ROOT_HOST}/docs" "./docs" -COPY "${MORPHEUS_ROOT_HOST}/examples" "./examples" -COPY "${MORPHEUS_ROOT_HOST}/models" "./models" -COPY "${MORPHEUS_ROOT_HOST}/scripts" "./scripts" -COPY "${MORPHEUS_ROOT_HOST}/*.md" "./" -COPY "${MORPHEUS_ROOT_HOST}/LICENSE" "./" +COPY "${MORPHEUS_ROOT_HOST}/conda/environments/runtime_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml" "./conda/environments/" # Mount Morpheus conda package build in `conda_bld_morpheus` RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target=/opt/conda/conda-bld \ @@ -254,10 +247,30 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target -c nvidia/label/dev \ -c pytorch \ -c defaults \ - morpheus &&\ + morpheus && \ + /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file \ + conda/environments/runtime_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml - /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file conda/environments/runtime_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml && \ - conda clean -afy +# ============ Stage: runtime ============ +# Setup container for runtime environment +FROM runtime_conda_create as runtime + +ARG MORPHEUS_ROOT_HOST + +# Only copy specific files/folders over that are necessary for runtime +COPY "${MORPHEUS_ROOT_HOST}/conda/environments/*.yaml" "./conda/environments/" +COPY "${MORPHEUS_ROOT_HOST}/docker" "./docker" +COPY "${MORPHEUS_ROOT_HOST}/docs" "./docs" +COPY "${MORPHEUS_ROOT_HOST}/examples" "./examples" +COPY "${MORPHEUS_ROOT_HOST}/models" "./models" +COPY "${MORPHEUS_ROOT_HOST}/scripts" "./scripts" +COPY "${MORPHEUS_ROOT_HOST}/*.md" "./" +COPY "${MORPHEUS_ROOT_HOST}/LICENSE" "./" + +RUN /opt/conda/bin/conda clean -afy && \ + # Ensure the conda-bld directory is indexed even if empty + mkdir -p /opt/conda/conda-bld && \ + /opt/conda/bin/conda index /opt/conda/conda-bld # Use morpheus by default CMD [ "morpheus" ] diff --git a/docker/build_conda_packages.sh b/docker/build_conda_packages.sh index 43e0b1bbba..006737eff0 100755 --- a/docker/build_conda_packages.sh +++ b/docker/build_conda_packages.sh @@ -53,7 +53,9 @@ DOCKER_EXTRA_ARGS=() BUILD_SCRIPT="${BUILD_SCRIPT} export CONDA_ARGS=\"${CONDA_ARGS[@]}\" ./ci/conda/recipes/run_conda_build.sh "$@" +EXIT_CODE=\$? chown -R ${CUR_UID}:${CUR_GID} .cache .conda-bld +exit \$EXIT_CODE " echo "Running conda build" @@ -62,4 +64,4 @@ echo "Running conda build" DOCKER_EXTRA_ARGS="${DOCKER_EXTRA_ARGS[@]}" ${SCRIPT_DIR}/run_container_dev.sh bash -c "${BUILD_SCRIPT}" echo "Conda packages have been built. Use the following to install into an environment:" -echo " mamba install -c file://$(realpath ${MORPHEUS_ROOT}/.conda-bld) -c nvidia -c rapidsai -c conda-forge $@" +echo " mamba install -c file://$(realpath ${MORPHEUS_ROOT}/.conda-bld) -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia -c nvidia/label/dev $@" diff --git a/docker/build_container_release.sh b/docker/build_container_release.sh index ee4028a89a..01d505c48a 100755 --- a/docker/build_container_release.sh +++ b/docker/build_container_release.sh @@ -26,5 +26,8 @@ export DOCKER_TARGET=${DOCKER_TARGET:-"runtime"} popd &> /dev/null +# Fetch data +"${SCRIPT_DIR}/../scripts/fetch_data.py" fetch docs examples models + # Call the general build script ${SCRIPT_DIR}/build_container.sh diff --git a/docker/run_container_dev.sh b/docker/run_container_dev.sh index e440763988..9a2db756af 100755 --- a/docker/run_container_dev.sh +++ b/docker/run_container_dev.sh @@ -45,4 +45,7 @@ docker run \ -ti \ ${DOCKER_ARGS} ${DOCKER_EXTRA_ARGS} \ ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} "${@:-bash}" -set +x + +{ EXIT_CODE=$?; set +x; } 2>/dev/null + +exit $EXIT_CODE diff --git a/docs/README.md b/docs/README.md index 28d0d2d0b4..469303430e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,3 +30,5 @@ conda env update --solver=libmamba -n morpheus --file conda/environments/dev_cud CMAKE_CONFIGURE_EXTRA_ARGS="-DMORPHEUS_BUILD_DOCS=ON" ./scripts/compile.sh --target morpheus_docs ``` Outputs to `build/docs/html` + +If the documentation build is unsuccessful, refer to the **Out of Date Build Cache** section in [Troubleshooting](./source/extra_info/troubleshooting.md) to troubleshoot. diff --git a/docs/source/cloud_deployment_guide.md b/docs/source/cloud_deployment_guide.md index ae64ff5bcb..4825ef2412 100644 --- a/docs/source/cloud_deployment_guide.md +++ b/docs/source/cloud_deployment_guide.md @@ -104,7 +104,7 @@ The Helm chart (`morpheus-ai-engine`) that offers the auxiliary components requi Follow the below steps to install Morpheus AI Engine: ```bash -helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-ai-engine-24.03.tgz --username='$oauthtoken' --password=$API_KEY --untar +helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-ai-engine-24.06.tgz --username='$oauthtoken' --password=$API_KEY --untar ``` ```bash helm install --set ngc.apiKey="$API_KEY" \ @@ -146,7 +146,7 @@ replicaset.apps/zookeeper-87f9f4dd 1 1 1 54s Run the following command to pull the Morpheus SDK Client (referred to as Helm chart `morpheus-sdk-client`) on to your instance: ```bash -helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-sdk-client-24.03.tgz --username='$oauthtoken' --password=$API_KEY --untar +helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-sdk-client-24.06.tgz --username='$oauthtoken' --password=$API_KEY --untar ``` #### Morpheus SDK Client in Sleep Mode @@ -184,7 +184,7 @@ kubectl -n $NAMESPACE exec sdk-cli-helper -- cp -RL /workspace/models /common The Morpheus MLflow Helm chart offers MLflow server with Triton plugin to deploy, update, and remove models from the Morpheus AI Engine. The MLflow server UI can be accessed using NodePort `30500`. Follow the below steps to install the Morpheus MLflow: ```bash -helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-mlflow-24.03.tgz --username='$oauthtoken' --password=$API_KEY --untar +helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-mlflow-24.06.tgz --username='$oauthtoken' --password=$API_KEY --untar ``` ```bash helm install --set ngc.apiKey="$API_KEY" \ @@ -402,7 +402,7 @@ To publish messages to a Kafka topic, we need to copy datasets to locations wher kubectl -n $NAMESPACE exec sdk-cli-helper -- cp -R /workspace/examples/data /common ``` -Refer to the [Morpheus CLI Overview](https://github.com/nv-morpheus/Morpheus/blob/branch-24.03/docs/source/basics/overview.rst) and [Building a Pipeline](https://github.com/nv-morpheus/Morpheus/blob/branch-24.03/docs/source/basics/building_a_pipeline.md) documentation for more information regarding the commands. +Refer to the [Morpheus CLI Overview](https://github.com/nv-morpheus/Morpheus/blob/branch-24.06/docs/source/basics/overview.rst) and [Building a Pipeline](https://github.com/nv-morpheus/Morpheus/blob/branch-24.06/docs/source/basics/building_a_pipeline.md) documentation for more information regarding the commands. > **Note**: Before running the example pipelines, ensure the criteria below are met: - Ensure models specific to the pipeline are deployed. @@ -782,8 +782,8 @@ kubectl -n $NAMESPACE exec deploy/broker -c broker -- kafka-topics.sh \ ## Additional Documentation For more information on how to use the Morpheus Python API to customize and run your own optimized AI pipelines, Refer to below documentation. -- [Morpheus Developer Guides](https://github.com/nv-morpheus/Morpheus/blob/branch-24.03/docs/source/developer_guide/guides.md) -- [Morpheus Pipeline Examples](https://github.com/nv-morpheus/Morpheus/tree/branch-24.03/examples) +- [Morpheus Developer Guides](https://github.com/nv-morpheus/Morpheus/blob/branch-24.06/docs/source/developer_guide/guides.md) +- [Morpheus Pipeline Examples](https://github.com/nv-morpheus/Morpheus/tree/branch-24.06/examples) ## Troubleshooting diff --git a/docs/source/conf.py b/docs/source/conf.py index 053e8214a8..709e8230e0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -172,6 +172,7 @@ "databricks.connect", "datacompy", "langchain", + "langchain_core", "merlin", "morpheus.cli.commands", # Dont document the CLI in Sphinx "nvtabular", diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 66ad68fcb6..1c9299f1a4 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -63,21 +63,13 @@ Review the unassigned issues, and find an issue to which you are comfortable con The following instructions are for developers who are getting started with the Morpheus repository. The Morpheus development environment is flexible (Docker, Conda and bare metal workflows) but has a high number of dependencies that can be difficult to set up. These instructions outline the steps for setting up a development environment inside a Docker container or on a host machine with Conda. -All of the following instructions assume several variables have been set: +All of the following instructions assume that the given variable has been defined: - `MORPHEUS_ROOT`: The Morpheus repository has been checked out at a location specified by this variable. Any non-absolute paths are relative to `MORPHEUS_ROOT`. - - `PYTHON_VER`: The desired Python version. Minimum required is `3.10` - - `RAPIDS_VER`: The desired RAPIDS version for all RAPIDS libraries including cuDF and RMM. If in doubt use `23.06` - - `TRITONCLIENT_VERSION`: The desired Triton client. If in doubt use `22.10` - - `CUDA_VER`: The desired CUDA version to use. If in doubt use `12.1` ### Clone the repository and pull large file data from Git LFS ```bash -export PYTHON_VER=3.10 -export RAPIDS_VER=23.06 -export TRITONCLIENT_VERSION=22.10 -export CUDA_VER=12.1 export MORPHEUS_ROOT=$(pwd)/morpheus git clone https://github.com/nv-morpheus/Morpheus.git $MORPHEUS_ROOT cd $MORPHEUS_ROOT @@ -144,26 +136,19 @@ This workflow utilizes a Docker container to set up most dependencies ensuring a Then once the container is started you will need to install some extra packages to enable launching Docker containers: ```bash ./external/utilities/docker/install_docker.sh - - # Install utils for checking output - apt install -y jq bc ``` 3. Compile Morpheus ```bash ./scripts/compile.sh ``` - This script will run both CMake Configure with default options and CMake build. -4. Install Morpheus - ```bash - pip install -e /workspace - ``` - Once Morpheus has been built, it can be installed into the current virtual environment. -5. [Run Morpheus](../getting_started.md#running-morpheus) + This script will run CMake Configure with default options, the CMake build and install Morpheus into the environment. + +4. [Run Morpheus](../getting_started.md#running-morpheus) ```bash morpheus run pipeline-nlp ... ``` - At this point, Morpheus can be fully used. Any changes to Python code will not require a rebuild. Changes to C++ code will require calling `./scripts/compile.sh`. Installing Morpheus is only required once per virtual environment. + At this point, Morpheus can be fully used. Any changes to Python code will not require a rebuild. Changes to C++ code will require calling `./scripts/compile.sh`. ### Build in a Conda Environment @@ -188,9 +173,6 @@ Note: These instructions assume the user is using `mamba` instead of `conda` sin 1. Set up env variables and clone the repo: ```bash - export PYTHON_VER=3.10 - export RAPIDS_VER=23.06 - export CUDA_VER=12.1 export MORPHEUS_ROOT=$(pwd)/morpheus git clone https://github.com/nv-morpheus/Morpheus.git $MORPHEUS_ROOT cd $MORPHEUS_ROOT diff --git a/docs/source/developer_guide/guides.md b/docs/source/developer_guide/guides.md index 2d141e96db..9e4fba5ff7 100644 --- a/docs/source/developer_guide/guides.md +++ b/docs/source/developer_guide/guides.md @@ -33,9 +33,9 @@ in both Python and C++. - [Simple C++ Stage](./guides/3_simple_cpp_stage.md) - [Creating a C++ Source Stage](./guides/4_source_cpp_stage.md) -> **Note**: The code for the above guides can be found in the `examples/developer_guide` directory of the Morpheus repository. To build the C++ examples, pass `-DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_PYTHON_PERFORM_INSTALL=ON` to CMake when building Morpheus. Users building Morpheus with the provided `scripts/compile.sh` script can do do by setting the `CMAKE_CONFIGURE_EXTRA_ARGS` environment variable: +> **Note**: The code for the above guides can be found in the `examples/developer_guide` directory of the Morpheus repository. To build the C++ examples, pass `-DMORPHEUS_BUILD_EXAMPLES=ON` to CMake when building Morpheus. Users building Morpheus with the provided `scripts/compile.sh` script can do do by setting the `CMAKE_CONFIGURE_EXTRA_ARGS` environment variable: > ```bash -> CMAKE_CONFIGURE_EXTRA_ARGS="-DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_PYTHON_PERFORM_INSTALL=ON" ./scripts/compile.sh +> CMAKE_CONFIGURE_EXTRA_ARGS="-DMORPHEUS_BUILD_EXAMPLES=ON" ./scripts/compile.sh > ``` ## Morpheus Modules diff --git a/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md b/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md index bbf46ae691..46ccd34446 100644 --- a/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md +++ b/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md @@ -45,6 +45,12 @@ limitations under the License. - [DFP Post Processing](#dfp-post-processing) - [Serialize](#serialize) - [Write to File](#write-to-file) + - [Running Example Modular DFP Pipelines](#running-example-modular-dfp-pipelines) + - [System requirements](#system-requirements) + - [Building the services](#building-the-services) + - [Downloading the example datasets](#downloading-the-example-datasets) + - [Run Morpheus pipeline](#run-morpheus-pipeline) + - [Output Fields](#output-fields) ## Introduction @@ -522,7 +528,7 @@ pip install s3fs python examples/digital_fingerprinting/fetch_example_data.py all ``` -### Morpheus Pipeline +### Run Morpheus pipeline From the `examples/digital_fingerprinting/production` dir, run: ```bash docker compose run morpheus_pipeline bash diff --git a/docs/source/developer_guide/guides/3_simple_cpp_stage.md b/docs/source/developer_guide/guides/3_simple_cpp_stage.md index 3b0982d21e..6b1fbd9339 100644 --- a/docs/source/developer_guide/guides/3_simple_cpp_stage.md +++ b/docs/source/developer_guide/guides/3_simple_cpp_stage.md @@ -17,9 +17,9 @@ limitations under the License. # Simple C++ Stage ## Building the Example -The code for this guide can be found in the `examples/developer_guide/3_simple_cpp_stage` directory of the Morpheus repository. There are two ways to build the example. The first is to build the examples along with Morpheus by passing the `-DMORPHEUS_BUILD_EXAMPLES=ON` and `-DMORPHEUS_PYTHON_PERFORM_INSTALL=ON` flags to cmake, for users using the `scripts/compile.sh` at the root of the Morpheus repo can do this by setting the `CMAKE_CONFIGURE_EXTRA_ARGS` environment variable: +The code for this guide can be found in the `examples/developer_guide/3_simple_cpp_stage` directory of the Morpheus repository. There are two ways to build the example. The first is to build the examples along with Morpheus by passing the `-DMORPHEUS_BUILD_EXAMPLES=ON` flag to cmake, for users using the `scripts/compile.sh` at the root of the Morpheus repo can do this by setting the `CMAKE_CONFIGURE_EXTRA_ARGS` environment variable: ```bash -CMAKE_CONFIGURE_EXTRA_ARGS="-DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_PYTHON_PERFORM_INSTALL=ON" ./scripts/compile.sh +CMAKE_CONFIGURE_EXTRA_ARGS="-DMORPHEUS_BUILD_EXAMPLES=ON" ./scripts/compile.sh ``` The second method is to build the example as a standalone project. From the root of the Morpheus repo execute: @@ -54,7 +54,7 @@ def supports_cpp_node(self): return True ``` -C++ message object declarations can be found in the header files that are located in the `morpheus/_lib/include/morpheus/messages` directory. For example, the `MessageMeta` class declaration is located in `morpheus/_lib/include/morpheus/messages/meta.hpp`. In code this would be included as: +C++ message object declarations can be found in the header files that are located in the `morpheus/_lib/include/morpheus/messages` directory. For example, the `MessageMeta` class declaration is located in `morpheus/_lib/include/morpheus/messages/meta.hpp`. Since this code is outside of the morpheus directory it would be included as: ```cpp #include @@ -89,6 +89,7 @@ While our Python implementation accepts messages of any type (in the form of Pyt To start with, we have our Morpheus and MRC-specific includes: ```cpp +#include #include // for MultiMessage #include // for Segment Builder #include // for Segment Object @@ -100,12 +101,10 @@ We'll want to define our stage in its own namespace. In this case, we will name ```cpp namespace morpheus_example { -// pybind11 sets visibility to hidden by default; we want to export our symbols -#pragma GCC visibility push(default) - using namespace morpheus; -class PassThruStage : public mrc::pymrc::PythonNode, std::shared_ptr> +// pybind11 sets visibility to hidden by default; we want to export our symbols +class MORPHEUS_EXPORT PassThruStage : public mrc::pymrc::PythonNode, std::shared_ptr> { public: using base_t = mrc::pymrc::PythonNode, std::shared_ptr>; @@ -119,7 +118,13 @@ class PassThruStage : public mrc::pymrc::PythonNode