Merge remote-tracking branch 'upstream/branch-24.06' into patch-12

.devcontainer/Dockerfile

@@ -15,4 +15,9 @@
 
 FROM rapidsai/devcontainers:23.12-cpp-mambaforge-ubuntu22.04 AS base
 
+ENV MORPHEUS_SUPPORT_DOCA=ON
+
+COPY ./docker/optional_deps docker/optional_deps
+RUN ./docker/optional_deps/doca.sh /tmp/doca
+
 ENV PATH="${PATH}:/workspaces/morpheus/.devcontainer/bin"

.devcontainer/devcontainer.json

@@ -18,6 +18,7 @@
     "build": {
         "dockerfile": "Dockerfile"
     },
+    "privileged": true,
 
     "hostRequirements": {
         "gpu": true
@@ -33,7 +34,8 @@
     ],
 
     "runArgs": [
-        "--network=morpheus"
+        "--network=morpheus",
+        "-v=/dev/hugepages:/dev/hugepages"
     ],
 
     "containerEnv": {
@@ -93,8 +95,7 @@
             "type": "bind",
             "source": "${localWorkspaceFolder}/.devcontainer/opt/morpheus",
             "target": "/opt/morpheus"
-        },
-
+        }
     ],
 
     "features": {

.devcontainer/docker/optional_deps/doca.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF}
+LINUX_DISTRO=${LINUX_DISTRO:-ubuntu}
+LINUX_VER=${LINUX_VER:-22.04}
+DOCA_VERSION=${DOCA_VERSION:-2.7.0}
+
+# Exit early if nothing to do
+if [[ ${MORPHEUS_SUPPORT_DOCA} != @(TRUE|ON) ]]; then
+   exit 0
+fi
+
+WORKING_DIR=$1
+
+echo "Installing DOCA using directory: ${WORKING_DIR}"
+
+DEB_DIR=${WORKING_DIR}/deb
+
+mkdir -p ${DEB_DIR}
+
+DOCA_OS_VERSION="ubuntu2204"
+DOCA_PKG_LINK="https://www.mellanox.com/downloads/DOCA/DOCA_v${DOCA_VERSION}/host/doca-host_${DOCA_VERSION}-204000-24.04-${DOCA_OS_VERSION}_amd64.deb"
+
+# Upgrade the base packages (diff between image and Canonical upstream repo)
+apt update -y
+apt upgrade -y
+
+# Install wget
+apt install -y --no-install-recommends wget
+
+wget -qO - ${DOCA_PKG_LINK} -O doca-host.deb
+apt install ./doca-host.deb
+apt update
+apt install -y doca-all
+apt install -y doca-gpu doca-gpu-dev
+
+# Now install the gdrcopy library according to: https://github.com/NVIDIA/gdrcopy
+GDRCOPY_DIR=${WORKING_DIR}/gdrcopy
+
+if [[ ! -d "${GDRCOPY_DIR}" ]] ; then
+    git clone https://github.com/NVIDIA/gdrcopy.git ${GDRCOPY_DIR}
+    cd ${GDRCOPY_DIR}
+else
+    cd ${GDRCOPY_DIR}
+    git pull https://github.com/NVIDIA/gdrcopy.git
+fi
+
+make lib lib_install

ci/conda/recipes/run_conda_build.sh

@@ -19,7 +19,7 @@ NUMARGS=$#
 ARGS=$*
 
 function hasArg {
-    (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+   (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
 }
 
 function get_version() {
@@ -99,6 +99,14 @@ CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvid
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}pytorch")
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}defaults")
 
+if [[ ${NUMARGS} == 0 ]]; then
+   echo -e "${r}ERROR: No arguments were provided. Please provide at least one package to build. Available packages:${x}"
+   echo -e "${r}   morpheus${x}"
+   echo -e "${r}   pydebug${x}"
+   echo -e "${r}Exiting...${x}"
+   exit 12
+fi
+
 if hasArg morpheus; then
    # Set GIT_VERSION to set the project version inside of meta.yaml
    export GIT_VERSION="$(get_version)"
@@ -110,10 +118,10 @@ if hasArg morpheus; then
 fi
 
 if hasArg pydebug; then
-  export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2)
+   export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2)
 
-  echo "Running conda-build for python-dbg..."
-  set -x
-  conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ./ci/conda/recipes/python-dbg
-  set +x
+   echo "Running conda-build for python-dbg..."
+   set -x
+   conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ./ci/conda/recipes/python-dbg
+   set +x
 fi

docker/build_conda_packages.sh

@@ -53,7 +53,9 @@ DOCKER_EXTRA_ARGS=()
 BUILD_SCRIPT="${BUILD_SCRIPT}
 export CONDA_ARGS=\"${CONDA_ARGS[@]}\"
 ./ci/conda/recipes/run_conda_build.sh "$@"
+EXIT_CODE=\$?
 chown -R ${CUR_UID}:${CUR_GID} .cache .conda-bld
+exit \$EXIT_CODE
 "
 
 echo "Running conda build"
@@ -62,4 +64,4 @@ echo "Running conda build"
 DOCKER_EXTRA_ARGS="${DOCKER_EXTRA_ARGS[@]}" ${SCRIPT_DIR}/run_container_dev.sh bash -c "${BUILD_SCRIPT}"
 
 echo "Conda packages have been built. Use the following to install into an environment:"
-echo "    mamba install -c file://$(realpath ${MORPHEUS_ROOT}/.conda-bld) -c nvidia -c rapidsai -c conda-forge $@"
+echo "    mamba install -c file://$(realpath ${MORPHEUS_ROOT}/.conda-bld) -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia -c nvidia/label/dev $@"

docker/optional_deps/doca.sh

@@ -19,7 +19,7 @@ set -e
 MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF}
 LINUX_DISTRO=${LINUX_DISTRO:-ubuntu}
 LINUX_VER=${LINUX_VER:-22.04}
-DOCA_VERSION=${DOCA_VERSION:-2.6.0}
+DOCA_VERSION=${DOCA_VERSION:-2.7.0}
 
 # Exit early if nothing to do
 if [[ ${MORPHEUS_SUPPORT_DOCA} != @(TRUE|ON) ]]; then
@@ -34,64 +34,21 @@ DEB_DIR=${WORKING_DIR}/deb
 
 mkdir -p ${DEB_DIR}
 
-DOCA_REPO_LINK="https://linux.mellanox.com/public/repo/doca/${DOCA_VERSION}"
-DOCA_REPO="${DOCA_REPO_LINK}/ubuntu22.04"
-DOCA_REPO_ARCH="x86_64"
-DOCA_UPSTREAM_REPO="${DOCA_REPO}/${DOCA_REPO_ARCH}"
+DOCA_OS_VERSION="ubuntu2204"
+DOCA_PKG_LINK="https://www.mellanox.com/downloads/DOCA/DOCA_v${DOCA_VERSION}/host/doca-host_${DOCA_VERSION}-204000-24.04-${DOCA_OS_VERSION}_amd64.deb"
 
 # Upgrade the base packages (diff between image and Canonical upstream repo)
 apt update -y
 apt upgrade -y
 
-# Cleanup apt
-rm -rf /var/lib/apt/lists/*
-apt autoremove -y
+# Install wget
+apt install -y --no-install-recommends wget
 
-# Configure DOCA Repository, and install packages
-apt update -y
-
-# Install wget & Add the DOCA public repository
-apt install -y --no-install-recommends wget software-properties-common gpg-agent
-wget -qO - ${DOCA_UPSTREAM_REPO}/GPG-KEY-Mellanox.pub | apt-key add -
-add-apt-repository "deb [trusted=yes] ${DOCA_UPSTREAM_REPO} ./"
-apt update -y
-
-# Install base-rt content
-apt install -y --no-install-recommends \
-    doca-gpu \
-    doca-gpu-dev \
-    doca-prime-runtime \
-    doca-prime-sdk \
-    doca-sdk \
-    dpcp \
-    flexio \
-    ibacm \
-    ibverbs-utils \
-    librdmacm1 \
-    libibnetdisc5 \
-    libibumad3 \
-    libibmad5 \
-    libopensm \
-    libopenvswitch \
-    libyara8 \
-    mlnx-tools \
-    ofed-scripts \
-    openmpi \
-    openvswitch-common \
-    openvswitch-switch \
-    srptools \
-    mlnx-ethtool \
-    mlnx-iproute2 \
-    python3-pyverbs \
-    rdma-core \
-    ucx \
-    yara
-
-    # Cleanup apt
-rm -rf /usr/lib/python3/dist-packages
-apt remove -y software-properties-common gpg-agent
-rm -rf /var/lib/apt/lists/*
-apt autoremove -y
+wget -qO - ${DOCA_PKG_LINK} -O doca-host.deb
+apt install ./doca-host.deb
+apt update
+apt install -y doca-all
+apt install -y doca-gpu doca-gpu-dev
 
 # Now install the gdrcopy library according to: https://github.com/NVIDIA/gdrcopy
 GDRCOPY_DIR=${WORKING_DIR}/gdrcopy

docker/run_container_dev.sh

@@ -45,4 +45,7 @@ docker run \
     -ti \
     ${DOCKER_ARGS} ${DOCKER_EXTRA_ARGS} \
     ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} "${@:-bash}"
-set +x
+
+{ EXIT_CODE=$?; set +x; } 2>/dev/null
+
+exit $EXIT_CODE

docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md

@@ -45,6 +45,12 @@ limitations under the License.
     - [DFP Post Processing](#dfp-post-processing)
     - [Serialize](#serialize)
     - [Write to File](#write-to-file)
+  - [Running Example Modular DFP Pipelines](#running-example-modular-dfp-pipelines)
+    - [System requirements](#system-requirements)
+    - [Building the services](#building-the-services)
+    - [Downloading the example datasets](#downloading-the-example-datasets)
+    - [Run Morpheus pipeline](#run-morpheus-pipeline)
+    - [Output Fields](#output-fields)
 
 ## Introduction
 
@@ -522,7 +528,7 @@ pip install s3fs
 python examples/digital_fingerprinting/fetch_example_data.py all
 ```
 
-### Morpheus Pipeline
+### Run Morpheus pipeline
 From the `examples/digital_fingerprinting/production` dir, run:
 ```bash
 docker compose run morpheus_pipeline bash

docs/source/developer_guide/guides/3_simple_cpp_stage.md

@@ -54,7 +54,7 @@ def supports_cpp_node(self):
     return True
 ```
 
-C++ message object declarations can be found in the header files that are located in the `morpheus/_lib/include/morpheus/messages` directory. For example, the `MessageMeta` class declaration is located in `morpheus/_lib/include/morpheus/messages/meta.hpp`. In code this would be included as:
+C++ message object declarations can be found in the header files that are located in the `morpheus/_lib/include/morpheus/messages` directory. For example, the `MessageMeta` class declaration is located in `morpheus/_lib/include/morpheus/messages/meta.hpp`. Since this code is outside of the morpheus directory it would be included as:
 
 ```cpp
 #include <morpheus/messages/meta.hpp>
@@ -89,6 +89,7 @@ While our Python implementation accepts messages of any type (in the form of Pyt
 To start with, we have our Morpheus and MRC-specific includes:
 
 ```cpp
+#include <morpheus/export.h>
 #include <morpheus/messages/multi.hpp>  // for MultiMessage
 #include <mrc/segment/builder.hpp>      // for Segment Builder
 #include <mrc/segment/object.hpp>       // for Segment Object
@@ -100,12 +101,10 @@ We'll want to define our stage in its own namespace. In this case, we will name
 ```cpp
 namespace morpheus_example {
 
-// pybind11 sets visibility to hidden by default; we want to export our symbols
-#pragma GCC visibility push(default)
-
 using namespace morpheus;
 
-class PassThruStage : public mrc::pymrc::PythonNode<std::shared_ptr<MultiMessage>, std::shared_ptr<MultiMessage>>
+// pybind11 sets visibility to hidden by default; we want to export our symbols
+class MORPHEUS_EXPORT PassThruStage : public mrc::pymrc::PythonNode<std::shared_ptr<MultiMessage>, std::shared_ptr<MultiMessage>>
 {
   public:
     using base_t = mrc::pymrc::PythonNode<std::shared_ptr<MultiMessage>, std::shared_ptr<MultiMessage>>;
@@ -119,7 +118,13 @@ class PassThruStage : public mrc::pymrc::PythonNode<std::shared_ptr<MultiMessage
 };
 ```
 
-We explicitly set the visibility for the stage object in the namespace to default. This is due to a pybind11 requirement for module implementations to default symbol visibility to hidden (`-fvisibility=hidden`). More details about this can be found in the [pybind11 documentation](https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes).
+We explicitly set the visibility for the stage object to default by importing:
+```cpp
+#include <morpheus/export.h>
+```
+Then adding `MORPHEUS_EXPORT`, which is defined in `/build/autogenerated/include/morpheus/export.h` and is compiler agnostic, to the definition of the stage object.  
+This is due to a pybind11 requirement for module implementations to default symbol visibility to hidden (`-fvisibility=hidden`). More details about this can be found in the [pybind11 documentation](https://pybind11.readthedocs.io/en/stable/faq.html#someclass-declared-with-greater-visibility-than-the-type-of-its-field-someclass-member-wattributes).
+Any object, struct, or function that is intended to be exported should have `MORPHEUS_EXPORT` included in the definition.
 
 For simplicity, we defined `base_t` as an alias for our base class type because the definition can be quite long. Our base class type also defines a few additional type aliases for us: `subscribe_fn_t`, `sink_type_t` and `source_type_t`. The `sink_type_t` and `source_type_t` aliases are shortcuts for the sink and source types that this stage will be reading and writing. In this case both the `sink_type_t` and `source_type_t` resolve to `std::shared_ptr<MultiMessage>`. `subscribe_fn_t` (read as "subscribe function type") is an alias for:
 
@@ -134,7 +139,7 @@ All Morpheus C++ stages receive an instance of an MRC Segment Builder and a name
 We will also define an interface proxy object to keep the class definition separated from the Python interface. This isn't strictly required, but it is a convention used internally by Morpheus. Our proxy object will define a static method named `init` which is responsible for constructing a `PassThruStage` instance and returning it wrapped in a `shared_ptr`. There are many common Python types that pybind11 [automatically converts](https://pybind11.readthedocs.io/en/latest/advanced/cast/overview.html#conversion-table) to their associated C++ types. The MRC `Builder` is a C++ object with Python bindings. However there are other instances such as checking for values of `None` where the casting from Python to C++ types is not automatic. The proxy interface object fulfills this need and is used to help insulate Python bindings from internal implementation details.
 
 ```cpp
-struct PassThruStageInterfaceProxy
+struct MORPHEUS_EXPORT PassThruStageInterfaceProxy
 {
     static std::shared_ptr<mrc::segment::Object<PassThruStage>> init(mrc::segment::Builder &builder,
                                                                      const std::string &name);
@@ -146,6 +151,7 @@ struct PassThruStageInterfaceProxy
 ```cpp
 #pragma once
 
+#include <morpheus/export.h>            // for exporting symbols
 #include <morpheus/messages/multi.hpp>  // for MultiMessage
 #include <mrc/segment/builder.hpp>      // for Segment Builder
 #include <mrc/segment/object.hpp>       // for Segment Object
@@ -156,12 +162,10 @@ struct PassThruStageInterfaceProxy
 
 namespace morpheus_example {
 
-// pybind11 sets visibility to hidden by default; we want to export our symbols
-#pragma GCC visibility push(default)
-
 using namespace morpheus;
 
-class PassThruStage : public mrc::pymrc::PythonNode<std::shared_ptr<MultiMessage>, std::shared_ptr<MultiMessage>>
+// pybind11 sets visibility to hidden by default; we want to export our symbols
+class MORPHEUS_EXPORT PassThruStage : public mrc::pymrc::PythonNode<std::shared_ptr<MultiMessage>, std::shared_ptr<MultiMessage>>
 {
   public:
     using base_t = mrc::pymrc::PythonNode<std::shared_ptr<MultiMessage>, std::shared_ptr<MultiMessage>>;
@@ -174,13 +178,12 @@ class PassThruStage : public mrc::pymrc::PythonNode<std::shared_ptr<MultiMessage
     subscribe_fn_t build_operator();
 };
 
-struct PassThruStageInterfaceProxy
+struct MORPHEUS_EXPORT PassThruStageInterfaceProxy
 {
     static std::shared_ptr<mrc::segment::Object<PassThruStage>> init(mrc::segment::Builder& builder,
                                                                      const std::string& name);
 };
 
-#pragma GCC visibility pop
 }  // namespace morpheus_example
 ```