Addressing comments on rollback PR # 2

1. Made rollback & bw compatibility run in parallel 2. Not logging full exception anymore if I'm raising it 3. removed pillow and numpy from requirements
marqo-ai · Nov 27, 2024 · eefdf0e · eefdf0e
1 parent 6fb0642
commit eefdf0e
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 61 deletions.
diff --git a/.github/workflows/backwards_compatibility_marqo_execution.yml b/.github/workflows/backwards_compatibility_marqo_execution.yml
@@ -19,6 +19,13 @@ on:
         description: 'To version image identifier is a unique identifier for the target Marqo image, which can either be a tag or a digest. It should contain complete qualified image name with tag or digest. For example: marqoai/marqo:abcd1234 or marqoai/marqo@sha256:1234567890abcdef. This is used to pull images from ECR.'
         required: true
         type: string
+      mode:
+        description: 'The mode in which the compatibility tests are to be run. Options: "backwards_compatibility", "rollback"'
+        options:
+          - backwards_compatibility
+          - rollback
+        required: true
+        type: choice
   workflow_dispatch:
     # from_version: Used as: the identifier for a workflow call, for logging purposes and for pulling image from DockerHub. We need to pass a version here: ex: 2.11.1
     # to_version: Used as: the identifier for a workflow call and for logging purposes. We cannot use this to pull images from ECR or DockerHub (as opposed to from_version) since the to_version image has not been released yet. We need to pass a version here: ex: 2.11.5
@@ -38,6 +45,13 @@ on:
         description: 'To version image identifier is a unique identifier for the target Marqo image, which can either be a tag or a digest. It should contain complete qualified image name with tag or digest. For example: marqoai/marqo:abcd1234 or marqoai/marqo@sha256:1234567890abcdef. This is used to pull images from ECR.'
         required: true
         type: string
+      mode:
+        description: 'The mode in which the compatibility tests are to be run. Options: "backwards_compatibility", "rollback"'
+        options:
+          - backwards_compatibility
+          - rollback
+        required: true
+        type: choice
 
 jobs:
   Start-Runner:
@@ -115,36 +129,22 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
 
-      # Step to run the backwards compatibility test
+      # Step to run the compatibility test. This step can run both backwards_compatibility and rollback tests, based on the MODE argument
       - name: Run backwards compatibility test
         id: run-backwards-compatibility
         env:
           FROM_VERSION: ${{ inputs.from_version || github.event.inputs.from_version }}
           TO_VERSION: ${{ inputs.to_version || github.event.inputs.to_version }}
           TO_IMAGE: ${{ inputs.to_image || github.event.inputs.to_image }}
+          MODE: ${{ inputs.mode || github.event.inputs.mode }}
         run: |
             export PYTHONPATH=${{ github.workspace }}:$PYTHONPATH
             python tests/backwards_compatibility_tests/compatibility_test_runner.py \
-            --mode=backwards_compatibility \
+            --mode "$MODE" \
             --from_version "$FROM_VERSION" \
             --to_version "$TO_VERSION" \
             --to_image "$TO_IMAGE" \
 
-      # Step to run rollback tests. Only runs if backwards compatibility tests have succeeded
-      - name: Run rollback tests
-        if: success()
-        env:
-          FROM_VERSION: ${{ inputs.from_version || github.event.inputs.from_version }}
-          TO_VERSION: ${{ inputs.to_version || github.event.inputs.to_version }}
-          TO_IMAGE: ${{ inputs.to_image || github.event.inputs.to_image }}
-        run: |
-          export PYTHONPATH=${{ github.workspace }}:$PYTHONPATH
-          python tests/backwards_compatibility_tests/compatibility_test_runner.py \
-          --mode=rollback \
-          --from_version "$FROM_VERSION" \
-          --to_version "$TO_VERSION" \
-          --to_image "$TO_IMAGE" \
-
   Stop-Runner:
     name: Stop self-hosted EC2 runner
     permissions:

diff --git a/.github/workflows/backwards_compatibility_marqo_orchestrator.yml b/.github/workflows/backwards_compatibility_marqo_orchestrator.yml
@@ -134,8 +134,8 @@ jobs:
           aws-secret-access-key: ${{ secrets.ECR_PUSHER_AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
-  run-execution-workflow:
-    # Job to trigger execution workflows for each version combination
+  run-backwards-compatibility-execution-workflow:
+    # Job to trigger execution workflows for backwards compatibility test for each version combination
     name: Run all execution workflows
     needs: [orchestrate, check-if-image-exists, build-and-push-image]
     if: always() && (needs.orchestrate.result == 'success')
@@ -157,4 +157,32 @@ jobs:
       # complete qualified image name with the image digest (i.e 424082663841.dkr.ecr.us-east-1.amazonaws.com/marqo-compatibility-tests@sha256:1234567890abcdef).
       # The image_identifier can either come from the check-if-image-exists (i.e in case the image already exists in ECR) job or the build-and-push-image (i.e in case the image was built and pushed to ECR) job.
       to_image: ${{ needs.check-if-image-exists.outputs.image_exists == 'true' && needs.check-if-image-exists.outputs.image_identifier 
-                                  || needs.build-and-push-image.outputs.image_identifier }}
+                                  || needs.build-and-push-image.outputs.image_identifier }}
+      mode: "backwards_compatibility"
+
+
+  run-rollback-tests-execution-workflow:
+    # Job to trigger execution workflows for rollback test for each version combination
+    name: Run all execution workflows
+    needs: [ orchestrate, check-if-image-exists, build-and-push-image ]
+    if: always() && (needs.orchestrate.result == 'success')
+    strategy:
+      matrix:
+        from_version: ${{ fromJson(needs.orchestrate.outputs.list) }}
+    uses: ./.github/workflows/backwards_compatibility_marqo_execution.yml
+    secrets: inherit
+    permissions:
+      contents: read # This permission is necessary to read repository contents
+      actions: write # Used by machulav/ec2-github-runner@v2 for managing self-hosted runners. The workflow needs to create and manage GitHub Actions runners on EC2
+      id-token: write # Used by aws-actions/configure-aws-credentials@v4. Required for AWS authentication and OIDC token management
+      checks: write # Used implicitly by GitHub Actions to report job statuses and create check runs
+      statuses: write # Used implicitly by GitHub Actions to report job statuses and create check runs
+    with:
+      from_version: ${{ matrix.from_version }}
+      to_version: ${{ needs.orchestrate.outputs.to_version }}
+      # Pass the image_identifier to the execution workflow. By image_identifier, we refer to the
+      # complete qualified image name with the image digest (i.e 424082663841.dkr.ecr.us-east-1.amazonaws.com/marqo-compatibility-tests@sha256:1234567890abcdef).
+      # The image_identifier can either come from the check-if-image-exists (i.e in case the image already exists in ECR) job or the build-and-push-image (i.e in case the image was built and pushed to ECR) job.
+      to_image: ${{ needs.check-if-image-exists.outputs.image_exists == 'true' && needs.check-if-image-exists.outputs.image_identifier
+        || needs.build-and-push-image.outputs.image_identifier }}
+      mode: "rollback"