CLOUDP-211733 - Pipeline and build process refactoring (#1427)

* Refactor pipeline * Update calls to pipeline * Update operator inventory * Cleanup DevConfig class * type hinting * Fix agent key * Dev config method renaming * Handle architectures and tags correctly * Pre-commit formatting * black formatting * Correct usage of --release flag * Refactor arg parsing and handle gh_run_id * Correct agent arg usage * Fix e2e building * Check that image_dev exists * Handle skip tag * Fix push manifest logic for e2e * Better handling of manifest creation * Black formatting * Remove old agent in Contributing and Makefile * Update pipeline.py * Improve function for CLI command * Fix usage of inventory tags * Updated Helm Charts * Revert helm changes * temporary disable workflow condition * Use the right release keys * Temporary disable if * Revert "Temporary disable if" This reverts commit bf8287c. * Revert "temporary disable workflow condition" This reverts commit 1a2ca3e. * Warning user if releasing e2e * black formatting
mongodb · Nov 28, 2023 · f3ff90f · f3ff90f
1 parent bf054bd
commit f3ff90f
Show file tree

Hide file tree

Showing 19 changed files with 334 additions and 608 deletions.
diff --git a/.action_templates/jobs/setup.yaml b/.action_templates/jobs/setup.yaml
@@ -4,8 +4,8 @@ setup:
     fail-fast: false
     matrix:
       include:
-        - pipeline-argument: operator-ubi
-        - pipeline-argument: version-post-start-hook-init
-        - pipeline-argument: readiness-probe-init
-        - pipeline-argument: agent-ubi
+        - pipeline-argument: mongodb-kubernetes-operator
+        - pipeline-argument: version-upgrade-hook
+        - pipeline-argument: readiness-probe
+        - pipeline-argument: agent
         - pipeline-argument: e2e
diff --git a/.action_templates/steps/build-and-push-development-images.yaml b/.action_templates/steps/build-and-push-development-images.yaml
@@ -1,6 +1,6 @@
 - name: Build and Push Images
   run: |
-    python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false --tag ${{ github.run_id }}
+    python pipeline.py --image-name ${{ matrix.pipeline-argument }}  --tag ${{ github.run_id }}
   env:
     MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json"
     version_id: "${{ github.run_id }}"
diff --git a/.github/workflows/e2e-dispatch.yml b/.github/workflows/e2e-dispatch.yml
@@ -40,10 +40,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - pipeline-argument: operator-ubi
-        - pipeline-argument: version-post-start-hook-init
-        - pipeline-argument: readiness-probe-init
-        - pipeline-argument: agent-ubi
+        - pipeline-argument: operator
+        - pipeline-argument: version-upgrade-hook
+        - pipeline-argument: readiness-probe
+        - pipeline-argument: agent
         - pipeline-argument: e2e
     steps:
     # template: .action_templates/steps/checkout.yaml
@@ -76,7 +76,7 @@ jobs:
     # template: .action_templates/steps/build-and-push-development-images.yaml
     - name: Build and Push Images
       run: |
-        python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false --tag ${{ github.run_id }}
+        python pipeline.py --image-name ${{ matrix.pipeline-argument }}  --tag ${{ github.run_id }}
       env:
         MONGODB_COMMUNITY_CONFIG: ${{ github.workspace }}/scripts/ci/config.json
         version_id: ${{ github.run_id }}

diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml
@@ -33,10 +33,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - pipeline-argument: operator-ubi
-        - pipeline-argument: version-post-start-hook-init
-        - pipeline-argument: readiness-probe-init
-        - pipeline-argument: agent-ubi
+        - pipeline-argument: operator
+        - pipeline-argument: version-upgrade-hook
+        - pipeline-argument: readiness-probe
+        - pipeline-argument: agent
         - pipeline-argument: e2e
     if: contains(github.event.pull_request.labels.*.name, 'dependencies') || contains(github.event.pull_request.labels.*.name,
       'safe-to-test')
@@ -78,7 +78,7 @@ jobs:
     # template: .action_templates/steps/build-and-push-development-images.yaml
     - name: Build and Push Images
       run: |
-        python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false --tag ${{ github.run_id }}
+        python pipeline.py --image-name ${{ matrix.pipeline-argument }}  --tag ${{ github.run_id }}
       env:
         MONGODB_COMMUNITY_CONFIG: ${{ github.workspace }}/scripts/ci/config.json
         version_id: ${{ github.run_id }}

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -40,10 +40,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - pipeline-argument: operator-ubi
-        - pipeline-argument: version-post-start-hook-init
-        - pipeline-argument: readiness-probe-init
-        - pipeline-argument: agent-ubi
+        - pipeline-argument: operator
+        - pipeline-argument: version-upgrade-hook
+        - pipeline-argument: readiness-probe
+        - pipeline-argument: agent
         - pipeline-argument: e2e
     if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/master'
       || (github.event.pull_request.head.repo.full_name == github.repository && github.actor
@@ -84,7 +84,7 @@ jobs:
     # template: .action_templates/steps/build-and-push-development-images.yaml
     - name: Build and Push Images
       run: |
-        python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release false --tag ${{ github.run_id }}
+        python pipeline.py --image-name ${{ matrix.pipeline-argument }}  --tag ${{ github.run_id }}
       env:
         MONGODB_COMMUNITY_CONFIG: ${{ github.workspace }}/scripts/ci/config.json
         version_id: ${{ github.run_id }}

diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml
@@ -12,16 +12,15 @@ jobs:
     strategy:
       matrix:
         include:
-          - pipeline-argument: operator-ubi
-            release-key: mongodb-kubernetes-operator
-          - pipeline-argument: version-post-start-hook-init
+          - pipeline-argument: operator
+            release-key: operator
+          - pipeline-argument: version-upgrade-hook
             release-key: version-upgrade-hook
-          - pipeline-argument: readiness-probe-init
+          - pipeline-argument: readiness-probe
             release-key: readiness-probe
-          - pipeline-argument: agent-ubi
-            release-key: mongodb-agent
-          - pipeline-argument: agent-ubuntu
-            release-key: mongodb-agent
+          - pipeline-argument: agent
+            release-key: agent
+
     steps:
       - name: Checkout Code
         uses: actions/checkout@v2
@@ -57,7 +56,7 @@ jobs:
 
       - name: Publish Image To Quay
         if: steps.release_status.outputs.OUTPUT == 'unreleased'
-        run: python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release true
+        run: python pipeline.py --image-name ${{ matrix.pipeline-argument }} --release
         env:
           MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json"
           AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"

diff --git a/.github/workflows/release-single-image.yml b/.github/workflows/release-single-image.yml
@@ -46,7 +46,7 @@ jobs:
 
       - name: Publish Image To Quay
         if: steps.release_status.outputs.OUTPUT == 'unreleased'
-        run: python pipeline.py --image-name ${{ github.event.inputs.pipeline-argument }} --release true
+        run: python pipeline.py --image-name ${{ github.event.inputs.pipeline-argument }} --release
         env:
           MONGODB_COMMUNITY_CONFIG: "${{ github.workspace }}/scripts/ci/config.json"
           AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"

diff --git a/Makefile b/Makefile
@@ -144,20 +144,19 @@ generate-env-file: ## generates a local-test.env for local testing
 ##@ Image
 
 operator-image: ## Build and push the operator image
-	python pipeline.py --image-name operator-ubi
+	python pipeline.py --image-name operator
 
 e2e-image: ## Build and push e2e test image
 	python pipeline.py --image-name e2e
 
 agent-image: ## Build and push agent image
-	python pipeline.py --image-name agent-ubuntu
-	python pipeline.py --image-name agent-ubi
+	python pipeline.py --image-name agent
 
 readiness-probe-image: ## Build and push readiness probe image
-	python pipeline.py --image-name readiness-probe-init
+	python pipeline.py --image-name readiness-probe
 
 version-upgrade-post-start-hook-image: ## Build and push version upgrade post start hook image
-	python pipeline.py --image-name version-post-start-hook-init
+	python pipeline.py --image-name version-upgrade-hook
 
 all-images: operator-image e2e-image agent-image readiness-probe-image version-upgrade-post-start-hook-image ## create all required images
 

diff --git a/docs/contributing.md b/docs/contributing.md
@@ -59,8 +59,7 @@ to be able to run properly. Create a json file with the following content:
   "operator_image": "mongodb-kubernetes-operator",
   "e2e_image": "community-operator-e2e",
   "version_upgrade_hook_image": "community-operator-version-upgrade-post-start-hook",
-  "agent_image_ubuntu": "mongodb-agent-dev",
-  "agent_image_ubi": "mongodb-agent-ubi-dev",
+  "agent_image": "mongodb-agent-ubi-dev",
   "readiness_probe_image": "mongodb-kubernetes-readiness",
   "s3_bucket": ""
 }
@@ -73,10 +72,8 @@ to be able to run properly. Create a json file with the following content:
 3. `operator_image` will be used as the name of the operator deployment, and the name of the operator image when build.
 4. `e2e_image` the name of e2e test image that will be built.
 5. `version_upgrade_hook_image` the name of the version upgrade post start hook image.
-6. `image_type` this can be either `ubi` or `ubuntu` and determines the distro of the images built. (currently only the agent image has multiple distros)
-7. `agent_image_ubuntu` the name of the ubuntu agent image.
-8. `agent_image_ubi` the name of the ubi agent image.
-9. `s3_bucket` the S3 bucket that Dockerfiles will be pushed to as part of the release process. Note: this is only required when running the release tasks locally.
+6. `agent_image` the name of the agent image.
+7. `s3_bucket` the S3 bucket that Dockerfiles will be pushed to as part of the release process. Note: this is only required when running the release tasks locally.
 
 
 You can set the `MONGODB_COMMUNITY_CONFIG` environment variable to be the absolute path of this file.

diff --git a/inventories/e2e-inventory.yaml b/inventories/e2e-inventory.yaml
@@ -7,7 +7,7 @@ images:
       context: .
       template_context: scripts/dev/templates
     inputs:
-      - e2e_image
+      - image
     platform: linux/arm64
     stages:
       - name: e2e-template
@@ -30,17 +30,17 @@ images:
           quay.expires-after: 48h
 
         output:
-          - registry: $(inputs.params.registry)/$(inputs.params.e2e_image)
+          - registry: $(inputs.params.registry)/$(inputs.params.image)
             tag: $(inputs.params.version_id)-arm64
-          - registry: $(inputs.params.registry)/$(inputs.params.e2e_image)
+          - registry: $(inputs.params.registry)/$(inputs.params.image)
             tag: latest-arm64
 
   - name: e2e-amd64
     vars:
       context: .
       template_context: scripts/dev/templates
     inputs:
-      - e2e_image
+      - image
     platform: linux/amd64
     stages:
       - name: e2e-template
@@ -63,8 +63,8 @@ images:
           quay.expires-after: 48h
 
         output:
-          - registry: $(inputs.params.registry)/$(inputs.params.e2e_image)
+          - registry: $(inputs.params.registry)/$(inputs.params.image)
             tag: $(inputs.params.version_id)-amd64
-          - registry: $(inputs.params.registry)/$(inputs.params.e2e_image)
+          - registry: $(inputs.params.registry)/$(inputs.params.image)
             tag: latest-amd64
 
diff --git a/inventories/operator-inventory.yaml b/inventories/operator-inventory.yaml
@@ -2,14 +2,14 @@ vars:
   registry: <registry>
 
 images:
-  - name: operator-ubi-amd64
+  - name: operator-amd64
     vars:
       context: .
       template_context: scripts/dev/templates/operator
 
     inputs:
-      - operator_image
-      - operator_image_dev
+      - image
+      - image_dev
 
     platform: linux/amd64
 
@@ -29,7 +29,7 @@ images:
           quay.expires-after: 48h
 
         output:
-        - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev)
+        - registry: $(inputs.params.registry)/$(inputs.params.image_dev)
           tag: $(inputs.params.version_id)-context-amd64
 
       - name: operator-template-dev
@@ -51,15 +51,15 @@ images:
           - version_id
 
         buildargs:
-          imagebase: $(inputs.params.registry)/$(inputs.params.operator_image_dev):$(inputs.params.version_id)-context-amd64
+          imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-amd64
 
         labels:
           quay.expires-after: 48h
 
         output:
-          - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev)
+          - registry: $(inputs.params.registry)/$(inputs.params.image_dev)
             tag: $(inputs.params.version_id)-amd64
-          - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev)
+          - registry: $(inputs.params.registry)/$(inputs.params.image_dev)
             tag: latest-amd64
 
 #
@@ -82,7 +82,7 @@ images:
           builder_image: $(inputs.params.builder_image)
 
         output:
-          - registry: $(inputs.params.registry)/$(inputs.params.operator_image)
+          - registry: $(inputs.params.registry)/$(inputs.params.image)
             tag: $(inputs.params.release_version)-context-amd64
 
       - name: operator-template-release
@@ -107,23 +107,23 @@ images:
         dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.release_version)
 
         buildargs:
-          imagebase: $(inputs.params.registry)/$(inputs.params.operator_image):$(inputs.params.release_version)-context-amd64
+          imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-amd64
 
         labels:
           quay.expires-after: Never
 
         output:
-          - registry: $(inputs.params.registry)/$(inputs.params.operator_image)
+          - registry: $(inputs.params.registry)/$(inputs.params.image)
             tag: $(inputs.params.release_version)-amd64
 
-  - name: operator-ubi-arm64
+  - name: operator-arm64
     vars:
       context: .
       template_context: scripts/dev/templates/operator
 
     inputs:
-      - operator_image
-      - operator_image_dev
+      - image
+      - image_dev
 
     platform: linux/arm64
 
@@ -143,7 +143,7 @@ images:
           quay.expires-after: 48h
 
         output:
-          - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev)
+          - registry: $(inputs.params.registry)/$(inputs.params.image_dev)
             tag: $(inputs.params.version_id)-context-arm64
 
       - name: operator-template-dev
@@ -165,15 +165,15 @@ images:
           - version_id
 
         buildargs:
-          imagebase: $(inputs.params.registry)/$(inputs.params.operator_image_dev):$(inputs.params.version_id)-context-arm64
+          imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-arm64
 
         labels:
           quay.expires-after: 48h
 
         output:
-          - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev)
+          - registry: $(inputs.params.registry)/$(inputs.params.image_dev)
             tag: $(inputs.params.version_id)-arm64
-          - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev)
+          - registry: $(inputs.params.registry)/$(inputs.params.image_dev)
             tag: latest-arm64
 
       #
@@ -196,7 +196,7 @@ images:
           builder_image: $(inputs.params.builder_image)
 
         output:
-          - registry: $(inputs.params.registry)/$(inputs.params.operator_image)
+          - registry: $(inputs.params.registry)/$(inputs.params.image)
             tag: $(inputs.params.release_version)-context-arm64
 
       - name: operator-template-release
@@ -221,11 +221,11 @@ images:
         dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.release_version)
 
         buildargs:
-          imagebase: $(inputs.params.registry)/$(inputs.params.operator_image):$(inputs.params.release_version)-context-arm64
+          imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-arm64
 
         labels:
           quay.expires-after: Never
 
         output:
-          - registry: $(inputs.params.registry)/$(inputs.params.operator_image)
+          - registry: $(inputs.params.registry)/$(inputs.params.image)
             tag: $(inputs.params.release_version)-arm64