From 0badf403b277e763158b985866ea28a62fcb0562 Mon Sep 17 00:00:00 2001 From: Anuradha Karuppiah Date: Thu, 5 Sep 2024 19:36:41 +0000 Subject: [PATCH] Add CI pipeline job to build the morpheus core conda package 1. The morpheus-core conda package is built for main/dev branches and when "conda-core-build" label is added to the PR. 2. Enabled upload of morpheus-core conda package. The conda package with all morpheus compoments (core+llm) can still be built via the usual triggers. Uploading that package remains disabled. Signed-off-by: Anuradha Karuppiah --- .github/workflows/ci_pipe.yml | 52 +++++++++++++++++++++++++++++++++-- .github/workflows/pr.yaml | 12 ++++++-- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml index 34dfbd38cd..fb008db4be 100644 --- a/.github/workflows/ci_pipe.yml +++ b/.github/workflows/ci_pipe.yml @@ -21,8 +21,16 @@ on: run_check: required: true type: boolean + conda_core_run_build: + description: 'Runs the stage to build the morpheus-core conda package' + required: true + type: boolean + conda_core_upload_label: + description: 'The label to use when uploading the morpheus-core conda package. Leave empty to disable uploading' + required: true + type: string conda_run_build: - description: 'Runs the conda-build stage to ensure the conda package builds successfully' + description: 'Runs the conda-build stage to build the conda package with all morpheus components' required: true type: boolean conda_upload_label: @@ -36,6 +44,8 @@ on: required: true type: string secrets: + CONDA_TOKEN: + required: true NGC_API_KEY: required: true @@ -198,8 +208,46 @@ jobs: shell: bash run: ./morpheus/ci/scripts/github/docs.sh + package-core: + name: Package Core + if: ${{ inputs.conda_core_run_build }} + needs: [check, documentation, test] + runs-on: linux-amd64-cpu16 + timeout-minutes: 60 + container: + credentials: + username: '$oauthtoken' + password: ${{ secrets.NGC_API_KEY }} + image: ${{ inputs.container }} + strategy: + fail-fast: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + lfs: false + path: 'morpheus' + fetch-depth: 0 + submodules: 'recursive' + + - name: Get AWS credentials using OIDC + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + role-duration-seconds: 43200 # 12h + + - name: Build morpheus-core conda package + shell: bash + env: + CONDA_TOKEN: "${{ secrets.CONDA_TOKEN }}" + SCRIPT_ARGS: "${{ inputs.conda_core_upload_label != '' && 'upload' || '' }}" + CONDA_PKG_LABEL: "${{ inputs.conda_core_upload_label }}" + run: ./morpheus/ci/scripts/github/conda_core.sh $SCRIPT_ARGS + package: - name: Package + name: Package All if: ${{ inputs.conda_run_build }} needs: [check, documentation, test] runs-on: linux-amd64-cpu16 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 42a261fdc0..f67c9872e7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -68,6 +68,7 @@ jobs: is_main_branch: ${{ github.ref_name == 'main' }} is_dev_branch: ${{ startsWith(github.ref_name, 'branch-') }} has_conda_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-build') || false }} + has_conda_core_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-core-build') || false }} has_skip_ci_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'skip-ci') || false }} pr_info: ${{ steps.get-pr-info.outcome == 'success' && steps.get-pr-info.outputs.pr-info || '' }} @@ -89,13 +90,18 @@ jobs: with: # Run checks for any PR branch run_check: ${{ fromJSON(needs.prepare.outputs.is_pr) }} - # Run conda-build for main/dev branches and PRs with the conda-build label + # Build morpheus-core conda package. This is done for main/dev branches and + # for PRs with the conda-core-build label + conda_core_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_core_build_label) }} + # Upload morpheus-core conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches + conda_core_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }} + # Run morpheus conda package, with all components. This is done for main/dev + # branches and for PRs with the conda-build label. conda_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_build_label) }} - # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches # Disable conda upload for now, once we have morpheus packages in conda forge set the value to - # !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' conda_upload_label: "" container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-240614 test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-240614 secrets: + CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }} NGC_API_KEY: ${{ secrets.NGC_API_KEY }}