Add CI pipeline job to build the morpheus core conda package

1. The morpheus-core conda package is built for main/dev branches and when "conda-core-build" label is added to the PR. 2. Enabled upload of morpheus-core conda package. The conda package with all morpheus compoments (core+llm) can still be built via the usual triggers. Uploading that package remains disabled. Signed-off-by: Anuradha Karuppiah <[email protected]>
nv-morpheus · Sep 5, 2024 · 0badf40 · 0badf40
1 parent fd27043
commit 0badf40
Show file tree

Hide file tree

Showing 2 changed files with 59 additions and 5 deletions.
diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml
@@ -21,8 +21,16 @@ on:
       run_check:
         required: true
         type: boolean
+      conda_core_run_build:
+        description: 'Runs the stage to build the morpheus-core conda package'
+        required: true
+        type: boolean
+      conda_core_upload_label:
+        description: 'The label to use when uploading the morpheus-core conda package. Leave empty to disable uploading'
+        required: true
+        type: string
       conda_run_build:
-        description: 'Runs the conda-build stage to ensure the conda package builds successfully'
+        description: 'Runs the conda-build stage to build the conda package with all morpheus components'
         required: true
         type: boolean
       conda_upload_label:
@@ -36,6 +44,8 @@ on:
         required: true
         type: string
     secrets:
+      CONDA_TOKEN:
+        required: true
       NGC_API_KEY:
         required: true
 
@@ -198,8 +208,46 @@ jobs:
         shell: bash
         run: ./morpheus/ci/scripts/github/docs.sh
 
+  package-core:
+    name: Package Core
+    if: ${{ inputs.conda_core_run_build }}
+    needs: [check, documentation, test]
+    runs-on: linux-amd64-cpu16
+    timeout-minutes: 60
+    container:
+      credentials:
+        username: '$oauthtoken'
+        password: ${{ secrets.NGC_API_KEY }}
+      image: ${{ inputs.container }}
+    strategy:
+      fail-fast: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          lfs: false
+          path: 'morpheus'
+          fetch-depth: 0
+          submodules: 'recursive'
+
+      - name: Get AWS credentials using OIDC
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+          role-duration-seconds: 43200 # 12h
+
+      - name: Build morpheus-core conda package
+        shell: bash
+        env:
+          CONDA_TOKEN: "${{ secrets.CONDA_TOKEN }}"
+          SCRIPT_ARGS: "${{ inputs.conda_core_upload_label != '' && 'upload' || '' }}"
+          CONDA_PKG_LABEL: "${{ inputs.conda_core_upload_label }}"
+        run: ./morpheus/ci/scripts/github/conda_core.sh $SCRIPT_ARGS
+
   package:
-    name: Package
+    name: Package All
     if: ${{ inputs.conda_run_build }}
     needs: [check, documentation, test]
     runs-on: linux-amd64-cpu16

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -68,6 +68,7 @@ jobs:
       is_main_branch: ${{ github.ref_name == 'main' }}
       is_dev_branch: ${{ startsWith(github.ref_name, 'branch-') }}
       has_conda_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-build') || false }}
+      has_conda_core_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-core-build') || false }}
       has_skip_ci_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'skip-ci') || false }}
       pr_info: ${{ steps.get-pr-info.outcome == 'success' && steps.get-pr-info.outputs.pr-info || '' }}
 
@@ -89,13 +90,18 @@ jobs:
     with:
       # Run checks for any PR branch
       run_check: ${{ fromJSON(needs.prepare.outputs.is_pr) }}
-      # Run conda-build for main/dev branches and PRs with the conda-build label
+      # Build morpheus-core conda package. This is done for main/dev branches and
+      # for PRs with the conda-core-build label
+      conda_core_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_core_build_label) }}
+      # Upload morpheus-core conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
+      conda_core_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
+      # Run morpheus conda package, with all components. This is done for main/dev
+      # branches and for PRs with the conda-build label.
       conda_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_build_label) }}
-      # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       # Disable conda upload for now, once we have morpheus packages in conda forge set the value to
-      # !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || ''
       conda_upload_label: ""
       container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-240614
       test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-240614
     secrets:
+      CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }}
       NGC_API_KEY: ${{ secrets.NGC_API_KEY }}