Skip to content

Commit

Permalink
Merge branch 'branch-24.06' of github.com:nv-morpheus/Morpheus into d…
Browse files Browse the repository at this point in the history
…evcontainer-updates
  • Loading branch information
cwharris committed May 16, 2024
2 parents b3bf022 + ee9d932 commit add0456
Show file tree
Hide file tree
Showing 148 changed files with 4,662 additions and 2,073 deletions.
4 changes: 4 additions & 0 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,7 @@ ENV SCCACHE_REGION="us-east-2"
ENV SCCACHE_BUCKET="rapids-sccache-devs"
ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai"
ENV HISTFILE="/home/coder/.cache/._bash_history"

ENV MORPHEUS_SUPPORT_DOCA=ON
COPY ./docker/optional_deps docker/optional_deps
RUN ./docker/optional_deps/doca.sh /tmp/doca
128 changes: 128 additions & 0 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// !/bin/bash
// SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
{
"name": "Morpheus",
"build": {
"dockerfile": "Dockerfile"
},
"privileged": true,

"hostRequirements": {
"gpu": true
},

"capAdd":[
"SYS_NICE",
"SYS_PTRACE"
],

"securityOpt": [
"seccomp=unconfined"
],

"runArgs": [
"--network=morpheus",
"-v=/dev/hugepages:/dev/hugepages"
],

"containerEnv": {
"HOST_MORPHEUS_ROOT": "${localWorkspaceFolder}",
"MORPHEUS_ROOT": "${containerWorkspaceFolder}",
"DEFAULT_CONDA_ENV": "morpheus",
"MAMBA_NO_BANNER": "1",
"CONDA_ALWAYS_YES": "true",
"VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
"DISPLAY": "${localEnv:DISPLAY:-}",
"XAUTHORITY": "${localEnv:XAUTHORITY:-}",
"XDG_SESSION_TYPE": "${localEnv:XDG_SESSION_TYPE:-}",
"XDG_RUNTIME_DIR": "${localEnv:XDG_RUNTIME_DIR:-}",
"DBUS_SESSION_BUS_ADDRESS": "${localEnv:DBUS_SESSION_BUS_ADDRESS:-}"
},

"initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/initialize-command.sh" ],

"remoteUser": "coder",
"mounts": [
{
"type": "bind",
"source": "/tmp/.X11-unix",
"target": "/tmp/.X11-unix"
},
{
"type": "bind",
"source": "${localEnv:XDG_RUNTIME_DIR}",
"target": "${localEnv:XDG_RUNTIME_DIR}"
},
{
"type": "bind",
"source": "/run/dbus/system_bus_socket",
"target": "/run/dbus/system_bus_socket"
},
{
"type":"bind",
"source": "/var/run/docker.sock",
"target": "/var/run/docker.sock"
},
{
"type": "bind",
"source": "${localWorkspaceFolder}/.cache/conda/envs",
"target": "/home/coder/.conda/envs"
},
{
"type": "bind",
"source": "${localWorkspaceFolder}/../.conda/pkgs",
"target": "/home/coder/.conda/pkgs"
},
{
"type":"bind",
"source": "${localWorkspaceFolder}/../.config", // parent folder because sister repos are sibling dirs
"target": "/home/coder/.config"
},
{
"type": "bind",
"source": "${localWorkspaceFolder}/.devcontainer/opt/morpheus",
"target": "/opt/morpheus"
}
],

"features": {
"ghcr.io/devcontainers/features/docker-from-docker": {},
"ghcr.io/devcontainers/features/dotnet:1": {
"version": "6.0",
"installUsingApt": false
}
},

"customizations": {
"vscode": {
"extensions": [
"eamodio.gitlens",
"llvm-vs-code-extensions.vscode-clangd",
"ms-python.python",
"ms-vscode.cpptools",
"ms-vscode.cmake-tools",
"nvidia.nsight-vscode-edition",
"twxs.cmake",
"xaver.clang-format"
],
"settings": {
"cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake",
"C_Cpp.intelliSenseEngine": "disabled",
"python.terminal.activateEnvironment": false
}
}
},
}
64 changes: 64 additions & 0 deletions .devcontainer/docker/optional_deps/doca.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF}
LINUX_DISTRO=${LINUX_DISTRO:-ubuntu}
LINUX_VER=${LINUX_VER:-22.04}
DOCA_VERSION=${DOCA_VERSION:-2.7.0}

# Exit early if nothing to do
if [[ ${MORPHEUS_SUPPORT_DOCA} != @(TRUE|ON) ]]; then
exit 0
fi

WORKING_DIR=$1

echo "Installing DOCA using directory: ${WORKING_DIR}"

DEB_DIR=${WORKING_DIR}/deb

mkdir -p ${DEB_DIR}

DOCA_OS_VERSION="ubuntu2204"
DOCA_PKG_LINK="https://www.mellanox.com/downloads/DOCA/DOCA_v${DOCA_VERSION}/host/doca-host_${DOCA_VERSION}-204000-24.04-${DOCA_OS_VERSION}_amd64.deb"

# Upgrade the base packages (diff between image and Canonical upstream repo)
apt update -y
apt upgrade -y

# Install wget
apt install -y --no-install-recommends wget

wget -qO - ${DOCA_PKG_LINK} -O doca-host.deb
apt install ./doca-host.deb
apt update
apt install -y doca-all
apt install -y doca-gpu doca-gpu-dev

# Now install the gdrcopy library according to: https://github.com/NVIDIA/gdrcopy
GDRCOPY_DIR=${WORKING_DIR}/gdrcopy

if [[ ! -d "${GDRCOPY_DIR}" ]] ; then
git clone https://github.com/NVIDIA/gdrcopy.git ${GDRCOPY_DIR}
cd ${GDRCOPY_DIR}
else
cd ${GDRCOPY_DIR}
git pull https://github.com/NVIDIA/gdrcopy.git
fi

make lib lib_install
20 changes: 14 additions & 6 deletions ci/conda/recipes/run_conda_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ NUMARGS=$#
ARGS=$*

function hasArg {
(( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
(( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
}

function get_version() {
Expand Down Expand Up @@ -99,6 +99,14 @@ CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvid
CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}pytorch")
CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}defaults")

if [[ ${NUMARGS} == 0 ]]; then
echo -e "${r}ERROR: No arguments were provided. Please provide at least one package to build. Available packages:${x}"
echo -e "${r} morpheus${x}"
echo -e "${r} pydebug${x}"
echo -e "${r}Exiting...${x}"
exit 12
fi

if hasArg morpheus; then
# Set GIT_VERSION to set the project version inside of meta.yaml
export GIT_VERSION="$(get_version)"
Expand All @@ -110,10 +118,10 @@ if hasArg morpheus; then
fi

if hasArg pydebug; then
export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2)
export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2)

echo "Running conda-build for python-dbg..."
set -x
conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ./ci/conda/recipes/python-dbg
set +x
echo "Running conda-build for python-dbg..."
set -x
conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ./ci/conda/recipes/python-dbg
set +x
fi
4 changes: 3 additions & 1 deletion docker/build_conda_packages.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ DOCKER_EXTRA_ARGS=()
BUILD_SCRIPT="${BUILD_SCRIPT}
export CONDA_ARGS=\"${CONDA_ARGS[@]}\"
./ci/conda/recipes/run_conda_build.sh "$@"
EXIT_CODE=\$?
chown -R ${CUR_UID}:${CUR_GID} .cache .conda-bld
exit \$EXIT_CODE
"

echo "Running conda build"
Expand All @@ -62,4 +64,4 @@ echo "Running conda build"
DOCKER_EXTRA_ARGS="${DOCKER_EXTRA_ARGS[@]}" ${SCRIPT_DIR}/run_container_dev.sh bash -c "${BUILD_SCRIPT}"

echo "Conda packages have been built. Use the following to install into an environment:"
echo " mamba install -c file://$(realpath ${MORPHEUS_ROOT}/.conda-bld) -c nvidia -c rapidsai -c conda-forge $@"
echo " mamba install -c file://$(realpath ${MORPHEUS_ROOT}/.conda-bld) -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia -c nvidia/label/dev $@"
63 changes: 10 additions & 53 deletions docker/optional_deps/doca.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ set -e
MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF}
LINUX_DISTRO=${LINUX_DISTRO:-ubuntu}
LINUX_VER=${LINUX_VER:-22.04}
DOCA_VERSION=${DOCA_VERSION:-2.6.0}
DOCA_VERSION=${DOCA_VERSION:-2.7.0}

# Exit early if nothing to do
if [[ ${MORPHEUS_SUPPORT_DOCA} != @(TRUE|ON) ]]; then
Expand All @@ -34,64 +34,21 @@ DEB_DIR=${WORKING_DIR}/deb

mkdir -p ${DEB_DIR}

DOCA_REPO_LINK="https://linux.mellanox.com/public/repo/doca/${DOCA_VERSION}"
DOCA_REPO="${DOCA_REPO_LINK}/ubuntu22.04"
DOCA_REPO_ARCH="x86_64"
DOCA_UPSTREAM_REPO="${DOCA_REPO}/${DOCA_REPO_ARCH}"
DOCA_OS_VERSION="ubuntu2204"
DOCA_PKG_LINK="https://www.mellanox.com/downloads/DOCA/DOCA_v${DOCA_VERSION}/host/doca-host_${DOCA_VERSION}-204000-24.04-${DOCA_OS_VERSION}_amd64.deb"

# Upgrade the base packages (diff between image and Canonical upstream repo)
apt update -y
apt upgrade -y

# Cleanup apt
rm -rf /var/lib/apt/lists/*
apt autoremove -y
# Install wget
apt install -y --no-install-recommends wget

# Configure DOCA Repository, and install packages
apt update -y

# Install wget & Add the DOCA public repository
apt install -y --no-install-recommends wget software-properties-common gpg-agent
wget -qO - ${DOCA_UPSTREAM_REPO}/GPG-KEY-Mellanox.pub | apt-key add -
add-apt-repository "deb [trusted=yes] ${DOCA_UPSTREAM_REPO} ./"
apt update -y

# Install base-rt content
apt install -y --no-install-recommends \
doca-gpu \
doca-gpu-dev \
doca-prime-runtime \
doca-prime-sdk \
doca-sdk \
dpcp \
flexio \
ibacm \
ibverbs-utils \
librdmacm1 \
libibnetdisc5 \
libibumad3 \
libibmad5 \
libopensm \
libopenvswitch \
libyara8 \
mlnx-tools \
ofed-scripts \
openmpi \
openvswitch-common \
openvswitch-switch \
srptools \
mlnx-ethtool \
mlnx-iproute2 \
python3-pyverbs \
rdma-core \
ucx \
yara

# Cleanup apt
rm -rf /usr/lib/python3/dist-packages
apt remove -y software-properties-common gpg-agent
rm -rf /var/lib/apt/lists/*
apt autoremove -y
wget -qO - ${DOCA_PKG_LINK} -O doca-host.deb
apt install ./doca-host.deb
apt update
apt install -y doca-all
apt install -y doca-gpu doca-gpu-dev

# Now install the gdrcopy library according to: https://github.com/NVIDIA/gdrcopy
GDRCOPY_DIR=${WORKING_DIR}/gdrcopy
Expand Down
5 changes: 4 additions & 1 deletion docker/run_container_dev.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,7 @@ docker run \
-ti \
${DOCKER_ARGS} ${DOCKER_EXTRA_ARGS} \
${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} "${@:-bash}"
set +x

{ EXIT_CODE=$?; set +x; } 2>/dev/null

exit $EXIT_CODE
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ limitations under the License.
- [DFP Post Processing](#dfp-post-processing)
- [Serialize](#serialize)
- [Write to File](#write-to-file)
- [Running Example Modular DFP Pipelines](#running-example-modular-dfp-pipelines)
- [System requirements](#system-requirements)
- [Building the services](#building-the-services)
- [Downloading the example datasets](#downloading-the-example-datasets)
- [Run Morpheus pipeline](#run-morpheus-pipeline)
- [Output Fields](#output-fields)

## Introduction

Expand Down Expand Up @@ -522,7 +528,7 @@ pip install s3fs
python examples/digital_fingerprinting/fetch_example_data.py all
```
### Morpheus Pipeline
### Run Morpheus pipeline
From the `examples/digital_fingerprinting/production` dir, run:
```bash
docker compose run morpheus_pipeline bash
Expand Down
Loading

0 comments on commit add0456

Please sign in to comment.