Port build of multi-platform dev docker images from 3.9.x to main (…

…#1125) * add support for ARM arch * add changelog entry * trigger dev image build on PR * remove docker driver from dev image workflow * use --push instead of load flag * update override flag * remove target from override flag * Build multiplatform image for amd64 and arm64 architectures on release * Set POETRY_VERSION as build argument in Dockerfile * Add workflow_dispatch to release-artifacts GitHub workflow to allow manual release of Docker images and PyPI packages --------- Co-authored-by: Anca Lita <[email protected]>
RasaHQ · Jul 11, 2024 · 6423418 · 6423418
1 parent 1f3fe9d
commit 6423418
Show file tree

Hide file tree

Showing 8 changed files with 299 additions and 258 deletions.
diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
@@ -10,11 +10,18 @@ on:
 
 env:
   IS_TAG_BUILD: ${{ startsWith(github.event.ref, 'refs/tags') }}
+  DEV_REPOSITORY: 329710836760.dkr.ecr.us-east-1.amazonaws.com/rasa-sdk-dev
+  AWS_REGION: us-east-1
 
 # SECRETS
 # - PYPI_TOKEN: publishing token for amn41 account, needs to be maintainer of
 #               RasaHQ/rasa-sdk on pypi (account credentials in 1password)
 
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+
 jobs:
   quality:
     name: Code Quality
@@ -128,208 +135,34 @@ jobs:
         with:
           dockerfile: "Dockerfile"
 
-  build_docker_image_set_env:
-    name: Prepare environment for Docker build
-    runs-on: ubuntu-22.04
-    outputs:
-      # Tag name used for intermediate images created during Docker image builds, e.g. 3886 - a PR number
-      image_tag: ${{ steps.set_output.outputs.image_tag }}
-      # Return 'true' if tag version is equal or higher than the latest tagged rasa-sdk version
-      is_newest_version: ${{ steps.rasa_sdk_get_version.outputs.is_newest_version }}
-    steps:
-      # Due to an issue with checking out a wrong commit, we make sure
-      # to checkout HEAD commit for a pull request.
-      # More details: https://github.com/actions/checkout/issues/299
-      - name: Checkout pull request HEAD commit instead of merge commit 🕝
-        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
-        if: github.event_name == 'pull_request'
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Checkout git repository 🕝
-        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
-        if: github.event_name != 'pull_request'
-
-      # Set environment variables for a pull request
-      #
-      # In this scenario, we've created a PR #1234
-      #
-      # Example output:
-      # IMAGE_TAG=1234
-      - name: Set environment variables - pull_request
-        if: github.event_name == 'pull_request' && env.IS_TAG_BUILD == 'false'
-        run: |
-          echo "IMAGE_TAG=${{ github.event.number }}" >> $GITHUB_ENV
-
-      # Set environment variables for a tag
-      #
-      # In this scenario, we've pushed the '2.4.1' tag
-      #
-      # Example output:
-      # IMAGE_TAG=2.4.1
-      - name: Set environment variables - push - tag
-        if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true'
-        run: |
-          TAG_NAME=${GITHUB_REF#refs/tags/}
-          echo "IMAGE_TAG=${TAG_NAME}" >> $GITHUB_ENV
-
-      # Set environment variables for a branch
-      #
-      # In this scenario, we've pushed changes into the main branch
-      #
-      # Example output:
-      # IMAGE_TAG=main
-      - name: Set environment variables - push - branch
-        if: github.event_name == 'push' && env.IS_TAG_BUILD == 'false'
-        run: |
-          BRANCH_NAME=${GITHUB_REF#refs/heads/}
-          echo "IMAGE_TAG=${BRANCH_NAME}" >> $GITHUB_ENV
-
-      - name: Set output
-        id: set_output
-        run: |
-          echo "image_tag=${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT
-
-      - name: Check if tag version is equal or higher than the latest tagged rasa-sdk version
-        id: rasa_sdk_get_version
-        if: env.IS_TAG_BUILD == 'true'
-        run: |
-          # Get latest tagged rasa-sdk version
-          git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
-          # Fetch branch history
-          git fetch --prune --unshallow
-          LATEST_TAGGED_NON_ALPHA_RASA_SDK_VERSION=$(git tag | sort -r -V | grep -E "^[0-9.]+$" | head -n1)
-          CURRENT_TAG=${GITHUB_REF#refs/tags/}
-          # Return 'true' if tag version is equal or higher than the latest tagged rasa-sdk version
-          IS_NEWEST_VERSION=$((printf '%s\n%s\n' "${LATEST_TAGGED_NON_ALPHA_RASA_SDK_VERSION}" "$CURRENT_TAG" \
-            | sort -V -C && echo true || echo false) || true)
-
-
-          if [[ "${IS_NEWEST_VERSION}" == "true" && "$CURRENT_TAG" =~ ^[0-9.]+$ ]]; then
-            echo "is_newest_version=true" >> $GITHUB_OUTPUT
-          else
-            echo "is_newest_version=false" >> $GITHUB_OUTPUT
-          fi
-
-  build_docker_image:
-    name: Build Docker image
+  rasa-sdk-dev-docker-image:
+    name: Build dev Docker image
     runs-on: ubuntu-22.04
-    needs: [quality, test, docker_linter, build_docker_image_set_env]
 
     steps:
-      # Due to an issue with checking out a wrong commit, we make sure
-      # to checkout HEAD commit for a pull request.
-      # More details: https://github.com/actions/checkout/issues/299
-      - name: Checkout pull request HEAD commit instead of merge commit 🕝
-        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
-        if: github.event_name == 'pull_request'
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Checkout git repository 🕝
-        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
-        if: github.event_name != 'pull_request'
+      - name: Check out code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v3.0.1
         with:
-          version: v0.5.1
-          driver: docker
-
-      - name: Set environment variables
-        run: |
-          echo "IMAGE_TAG=${{ needs.build_docker_image_set_env.outputs.image_tag }}" >> $GITHUB_ENV
-
-      - name: Build Docker image
-        run: |
-          docker buildx bake --load
-
-      - name: Login to Docker Hub Registry 🔢
-        if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true' && github.repository == 'RasaHQ/rasa-sdk'
-        run: echo ${{ secrets.DOCKER_HUB_PASSWORD }} | docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} --password-stdin || true
-
-      - name: Push Docker image 📦
-        if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true' && github.repository == 'RasaHQ/rasa-sdk'
-        run: |
-          IS_NEWEST_VERSION=${{ needs.build_docker_image_set_env.outputs.is_newest_version }}
-
-          # Push image
-          docker push rasa/rasa-sdk:${IMAGE_TAG}
-
-          # Tag the image as latest
-          if [[ "${IS_NEWEST_VERSION}" == "true" ]]; then
-            docker tag rasa/rasa-sdk:${IMAGE_TAG} rasa/rasa-sdk:latest
-            docker push rasa/rasa-sdk:latest
-          fi
+          role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
+          aws-region: ${{ env.AWS_REGION }}
 
-  deploy:
-    name: Deploy to PyPI
-    runs-on: ubuntu-22.04
-
-    # deploy will only be run when there is a tag available
-    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
-    needs: [quality, test, build_docker_image] # only run after all other stages succeeded
-
-    steps:
-      - name: Checkout git repository 🕝
-        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
-
-      - name: Set up Python 3.10 🐍
-        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435
-        with:
-          python-version: '3.10'
-
-      - name: Read Poetry Version 🔢
-        run: |
-          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
-        shell: bash
-
-      - name: Install poetry 🦄
-        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
         with:
-          poetry-version: ${{ env.POETRY_VERSION }}
+          mask-password: "true"
 
-      - name: Build ⚒️ Distributions
-        run: poetry build
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
 
-      - name: Publish to PyPI 📦
-        uses: pypa/gh-action-pypi-publish@bea5cda687c2b79989126d589ef4411bedce0195
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_TOKEN }}
-          skip_existing: true
-
-      - name: Publish Release Notes 🗞
-        if: env.IS_TAG_BUILD
-        env:
-          GITHUB_TAG: ${{ github.ref }}
-          GITHUB_REPO_SLUG: ${{ github.repository }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          GITHUB_TAG=${GITHUB_TAG/refs\/tags\//}
-          sudo apt-get update
-          sudo apt-get -y install pandoc
-          pip install -U github3.py pep440_version_utils
-          python3 ${GITHUB_WORKSPACE}/scripts/publish_gh_release_notes.py
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c  # v3.1.0
 
-      - name: Get RASA SDK Version
-        env:
-          RASA_SDK_VERSION: ${{ github.ref }}
+      - name: Build and push docker image to AWS
         run: |
-          echo "RASA_SDK_VERSION=${RASA_SDK_VERSION/refs\/tags\//}" >> $GITHUB_ENV
-
-      - name: Notify Slack 💬
-        if: env.IS_TAG_BUILD && success()
-        env:
-          SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_RELEASE_WEBHOOK }}
-        uses: Ilshidur/[email protected]
-        with:
-          args: "⚡ New *Rasa SDK* version ${{ env.RASA_SDK_VERSION }} has been released! Changelog: https://github.com/RasaHQ/rasa-sdk/blob/${{ env.RASA_SDK_VERSION }}/CHANGELOG.mdx"
-
-      - name: Notify Slack of Failure ⛔
-        if: env.IS_TAG_BUILD && failure()
-        env:
-          SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_DEV_TRIBE_WEBHOOK }}
-        uses: Ilshidur/[email protected]
-        with:
-          args: "⛔️ *Rasa SDK* version ${{ env.RASA_SDK_VERSION }} could not be released 😱 GitHub Actions: https://github.com/RasaHQ/rasa-sdk/actions?query=branch%3A${{ env.RASA_SDK_VERSION }}"
+          IMAGE_NAME=${{ env.DEV_REPOSITORY }} \
+          IMAGE_TAG=pr${{ github.event.number }} \
+          make build-and-push-multi-platform-docker
diff --git a/.github/workflows/pr-cleanup.yml b/.github/workflows/pr-cleanup.yml
@@ -0,0 +1,51 @@
+name: CI On Pull Request Closure
+on:
+  pull_request:
+    types: [closed]
+
+env:
+  AWS_REGION: us-east-1
+
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+jobs:  
+  aws-ecr:
+    # Skip any PR created by dependabot to avoid permission issues
+    if: (github.actor != 'dependabot[bot]')
+    name: Delete PR Docker Images
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 #v3.0.1
+        with:
+          role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d #v1.6
+        with:
+          mask-password: "true"
+
+      - name: Remove docker images from all ECR repositories
+        run: |
+          REPO_NAMES=("rasa-sdk-dev")
+          SEARCH_STRING="pr${{ github.event.number }}"
+          # Iterate through each repository
+          for REPO_NAME in "${REPO_NAMES[@]}"
+          do
+            # List all image tags in the repository that match the search string
+            # Escape empty strings and null character
+            IMAGES=$(aws ecr list-images --repository-name $REPO_NAME --query "imageIds[?imageTag!=null&&imageTag!=''&&contains(imageTag, '$SEARCH_STRING')].imageTag" --output text)
+            
+            # Print the image tags
+            echo "Images to be deleted in repository $REPO_NAME:"
+            echo "$IMAGES"
+            # Delete each image by its tag
+            for IMAGE in $IMAGES
+            do
+              aws ecr batch-delete-image --repository-name $REPO_NAME --image-ids imageTag=$IMAGE
+            done
+          done
diff --git a/...b/workflows/rasa-sdk-dev-docker-image.yml → .github/workflows/pr-merged.yml b/...b/workflows/rasa-sdk-dev-docker-image.yml → .github/workflows/pr-merged.yml
@@ -1,10 +1,9 @@
-name: Create Dev Docker Images
+name: Pull Request Merged to Main
 
 on:
-  schedule:
-    # Run cron job at 8AM Monday to Sunday.
-    - cron: '0 8 * * *'
-  workflow_dispatch:
+  push:
+    branches:
+      - main
 
 env:
   AWS_REGION: us-east-1
@@ -20,18 +19,13 @@ permissions:
 
 jobs:
   rasa-sdk-dev-docker-image:
-    name: rasa-sdk dev docker image
+    name: Build Dev Docker Image and Push to AWS
     runs-on: ubuntu-22.04
 
     steps:
       - name: Check out code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-      - name: Read Poetry Version 🔢
-        run: |
-          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
-        shell: bash
-
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v3.0.1
         with:
@@ -44,17 +38,14 @@ jobs:
         with:
           mask-password: "true"
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c
-        with:
-          version: v0.5.1
-          driver: docker
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
 
-      - name: Build docker image
-        run: |
-          docker buildx bake --load
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c  # v3.1.0
 
-      - name: Tag and push docker image to AWS
+      - name: Build and push docker image to AWS
         run: |
-          docker tag rasa/rasa-sdk:main $REPOSITORY:latest
-          docker push $REPOSITORY:latest
+          IMAGE_NAME=${{ env.REPOSITORY }} \
+          IMAGE_TAG=latest \
+          make build-and-push-multi-platform-docker