[RELEASE] Morpheus v24.03.00 (#1586)

<!--
SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION &
AFFILIATES. All rights reserved.
SPDX-License-Identifier: Apache-2.0

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

## ❄️ Code freeze for `branch-24.03` and `v24.03` release

### What does this mean?
Only critical/hotfix level issues should be merged into `branch-24.03`
until release (merging of this PR).

All other development PRs should be retargeted towards the next release
branch: `branch-24.06`.

### What is the purpose of this PR?
- Update documentation
- Allow testing for the new release
- Enable a means to merge `branch-24.03` into `main` for the release

.clang-tidy

@@ -5,7 +5,19 @@
 # Disable modernize-use-trailing-return-type (too many hits)
 # Disable modernize-use-nodiscard (why isnt [[nodiscard]] the default?)
 # Enable identifier in TODO comments
-Checks: -*,readability-identifier-naming,modernize-*,-modernize-use-trailing-return-type,-modernize-use-nodiscard,google-readability-todo
+Checks: >
+  -*,
+  readability-identifier-naming,
+  modernize-*,
+  -modernize-use-trailing-return-type,
+  -modernize-use-nodiscard,
+  google-readability-todo
+
+WarningsAsErrors: >
+  *,
+  -clang-diagnostic-unused-command-line-argument,
+  -Wno-ignored-optimization-argument,
+  -Qunused-arguments
 
 #WarningsAsErrors: '*'
 HeaderFilterRegex: '.*\/include\/morpheus\/.*'

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rapidsai/devcontainers:23.10-cpp-cuda11.8-mambaforge-ubuntu22.04 AS base
+FROM rapidsai/devcontainers:23.12-cpp-cuda12.1-mambaforge-ubuntu22.04 AS base
 
 ENV PATH="${PATH}:/workspaces/morpheus/.devcontainer/bin"

.devcontainer/README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 
 Licensed under the Apache License, Version 2.0 (the "License");

.devcontainer/devcontainer.json

@@ -99,7 +99,10 @@
 
     "features": {
         "ghcr.io/devcontainers/features/docker-from-docker": {},
-        "ghcr.io/devcontainers/features/dotnet:1": {}
+        "ghcr.io/devcontainers/features/dotnet:1": {
+            "version": "6.0",
+            "installUsingApt": false
+        }
     },
 
     "customizations": {
@@ -116,7 +119,6 @@
 			],
             "settings": {
                 "cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake",
-                "cmake.languageSupport.dotnetPath": "/usr/bin/dotnet",
                 "C_Cpp.intelliSenseEngine": "disabled",
                 "python.terminal.activateEnvironment": false
             }

.devcontainer/docker-compose.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.devcontainer/initialize-command.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.devcontainer/opt/morpheus/bin/post-attach-command.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,6 +27,6 @@ ENV_NAME=${ENV_NAME:-morpheus}
 sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;
 
 if conda_env_find "${ENV_NAME}" ; \
-then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml --prune; \
-else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml; \
+then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml --prune; \
+else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml; \
 fi

.devcontainer/opt/morpheus/bin/update-content-command.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.devcontainer/opt/morpheus/etc/enable_conda_libstd_pretty_print.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/ISSUE_TEMPLATE/config.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/ISSUE_TEMPLATE/documentation_request_correction.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/ISSUE_TEMPLATE/documentation_request_new.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/copy-pr-bot.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/ops-bot.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,3 +20,4 @@ auto_merger: true
 branch_checker: true
 label_checker: true
 release_drafter: true
+forward_merger: true

.github/workflows/ci_pipe.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,6 +21,14 @@ on:
       run_check:
         required: true
         type: boolean
+      conda_run_build:
+        description: 'Runs the conda-build stage to ensure the conda package builds successfully'
+        required: true
+        type: boolean
+      conda_upload_label:
+        description: 'The label to use when uploading the conda package. Leave empty to disable uploading'
+        required: true
+        type: string
       container:
         required: true
         type: string
@@ -34,7 +42,7 @@ on:
 env:
   CHANGE_TARGET: "${{ github.base_ref }}"
   CUDA_PATH: "/usr/local/cuda/"
-  CUDA_VER: "11.8"
+  CUDA_VER: "12.1"
   GH_TOKEN: "${{ github.token }}"
   GIT_COMMIT: "${{ github.sha }}"
   MORPHEUS_ROOT: "${{ github.workspace }}/morpheus"
@@ -132,7 +140,6 @@ jobs:
       env:
         NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
         PARALLEL_LEVEL: '10'
-        MERGE_EXAMPLES_YAML: '1'
     strategy:
       fail-fast: true
 
@@ -164,8 +171,6 @@ jobs:
         username: '$oauthtoken'
         password: ${{ secrets.NGC_API_KEY }}
       image: ${{ inputs.container }}
-      env:
-        MERGE_DOCS_YAML: '1'
     strategy:
       fail-fast: true
 
@@ -186,3 +191,40 @@ jobs:
       - name: build_docs
         shell: bash
         run: ./morpheus/ci/scripts/github/docs.sh
+
+  package:
+    name: Package
+    if: ${{ inputs.conda_run_build }}
+    needs: [check, documentation, test]
+    runs-on: linux-amd64-cpu16
+    timeout-minutes: 60
+    container:
+      credentials:
+        username: '$oauthtoken'
+        password: ${{ secrets.NGC_API_KEY }}
+      image: ${{ inputs.container }}
+    strategy:
+      fail-fast: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          lfs: false
+          path: 'morpheus'
+          fetch-depth: 0
+
+      - name: Get AWS credentials using OIDC
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+          role-duration-seconds: 43200 # 12h
+
+      - name: conda
+        shell: bash
+        env:
+          CONDA_TOKEN: "${{ secrets.CONDA_TOKEN }}"
+          SCRIPT_ARGS: "${{ inputs.conda_upload_label != '' && 'upload' || '' }}"
+          CONDA_PKG_LABEL: "${{ inputs.conda_upload_label }}"
+        run: ./morpheus/ci/scripts/github/conda.sh $SCRIPT_ARGS

.github/workflows/label-external-issues.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/workflows/pr.yaml

@@ -0,0 +1,101 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build pull request
+
+on:
+  push:
+    branches:
+      - 'pull-request/**'
+      - 'branch-*'
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
+permissions:
+  actions: none
+  checks: none
+  contents: read
+  deployments: none
+  discussions: none
+  id-token: write
+  issues: none
+  packages: read
+  pages: none
+  pull-requests: read
+  repository-projects: none
+  security-events: none
+  statuses: none
+
+jobs:
+
+  pr-builder:
+    needs:
+      - prepare
+      - checks
+      - ci_pipe
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+
+  prepare:
+    # Executes the get-pr-info action to determine if the PR has the skip-ci label, if the action fails we assume the
+    # PR does not have the label
+    name: Prepare
+    runs-on: ubuntu-latest
+    container:
+      image: rapidsai/ci-conda:latest
+    steps:
+      - name: Get PR Info
+        id: get-pr-info
+        uses: rapidsai/shared-action-workflows/[email protected]
+        if: ${{ startsWith(github.ref_name, 'pull-request/') }}
+    outputs:
+      is_pr: ${{ startsWith(github.ref_name, 'pull-request/') }}
+      is_main_branch: ${{ github.ref_name == 'main' }}
+      is_dev_branch: ${{ startsWith(github.ref_name, 'branch-') }}
+      has_conda_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-build') || false }}
+      has_skip_ci_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'skip-ci') || false }}
+      pr_info: ${{ steps.get-pr-info.outcome == 'success' && steps.get-pr-info.outputs.pr-info || '' }}
+
+  checks:
+    needs: [prepare]
+    # Only run the CI pipeline if the PR does not have the skip-ci label and we are on a PR branch
+    if: ${{ !fromJSON(needs.prepare.outputs.has_skip_ci_label) && fromJSON(needs.prepare.outputs.is_pr )}}
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      enable_check_generated_files: false
+
+  ci_pipe:
+    name: CI Pipeline
+    needs: [prepare]
+    uses: ./.github/workflows/ci_pipe.yml
+    # Only run the CI pipeline if the PR does not have the skip-ci label
+    if: ${{ ! fromJSON(needs.prepare.outputs.has_skip_ci_label) }}
+    with:
+      # Run checks for any PR branch
+      run_check: ${{ fromJSON(needs.prepare.outputs.is_pr) }}
+      # Run conda-build for main/dev branches and PRs with the conda-build label
+      conda_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_build_label) }}
+      # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
+      # Disable conda upload for now, once we have morpheus packages in conda forge set the value to
+      # !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || ''
+      conda_upload_label: ""
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-240221
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-240221
+    secrets:
+      NGC_API_KEY: ${{ secrets.NGC_API_KEY }}

.github/workflows/project_automation_set_in_progress.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/workflows/project_automation_set_in_review.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/workflows/project_automation_set_roadmap.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/workflows/project_automation_sync_pr_issues.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.github/workflows/project_automation_track_pr_dates.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");