Fix root folder reference (#84)

astronomer · Sep 30, 2024 · 6e70894 · 6e70894
1 parent b598870
commit 6e70894
Show file tree

Hide file tree

Showing 5 changed files with 181 additions and 15 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -100,11 +100,96 @@ jobs:
           DEPLOYMENT_ID=$(astro deployment create --deployment-file e2e-setup/deployment-templates/${{ matrix.deployment }} | yq e '.deployment.metadata.deployment_id' -)
           echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT
 
+  default-deploy-tests:
+    name: Default Deploy Test
+    runs-on: ubuntu-latest
+    needs: [create-test-deployments]
+    strategy:
+      max-parallel: 1
+      matrix:
+        deployment_id:
+          ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"]
+        deploy_type: [dags, image-and-dags]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get Astro Environment Info
+        id: get-astro-env-info
+        uses: ./.github/workflows/e2e/get_astro_env_info
+        with:
+          input_workspace_id: ${{ github.event.inputs.workspace_id }}
+          input_organization_id: ${{ github.event.inputs.org_id }}
+          input_astronomer_host: ${{ github.event.inputs.astronomer_host }}
+          input_astro_api_token: ${{ github.event.inputs.token }}
+          secret_workspace_id: ${{ secrets.WORKSPACE_ID }}
+          secret_organization_id: ${{ secrets.ORGANIZATION_ID }}
+          secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
+          secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
+
+      - name: Install dependencies
+        run: |
+
+          sudo apt-get install jq
+
+          # we need to pre-install the CLI to set the context
+          curl -sSL https://install.astronomer.io | sudo bash -s
+
+      - name: Mock git commands
+        run: |
+
+          # if deploy_type is dags, then use dags-deploy-git.sh as mock otherwise use image-deploy-git.sh as mock
+          if [[ "${{ matrix.deploy_type }}" == "dags" ]]; then
+            mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git
+          else
+            mv e2e-setup/mocks/image-deploy-git.sh /usr/local/bin/git
+          fi
+          chmod +x /usr/local/bin/git
+          cp /usr/local/bin/astro /usr/local/bin/astro-original
+          mv e2e-setup/mocks/astro-deploy.sh /usr/local/bin/astro
+          chmod +x /usr/local/bin/astro
+
+      - name: Set CLI context
+        run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Get Deployment Info Before Test
+        id: get-deployment-before
+        uses: ./.github/workflows/e2e/get_deployment_info
+        with:
+          deployment_id: ${{ matrix.deployment_id }}
+          organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
+          astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
+          astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Run Deploy Action
+        uses: ./
+        with:
+          deployment-id: ${{ matrix.deployment_id }}
+          workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }}
+
+      - name: Get Deployment Info After Test
+        id: get-deployment-after
+        uses: ./.github/workflows/e2e/get_deployment_info
+        with:
+          deployment_id: ${{ matrix.deployment_id }}
+          organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
+          astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
+          astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Validate Deploy Action
+        uses: ./.github/workflows/e2e/validate_deployment
+        with:
+          is_dag_only_deploy: ${{ matrix.deploy_type == 'dags' }}
+          dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
+          image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
+          dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
+          image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
+
   # DAG Deploy test would test the DAG only deploy functionality in deploy action
   dag-deploy-test:
     name: DAG Deploy Test
     runs-on: ubuntu-latest
-    needs: [create-test-deployments]
+    needs: [default-deploy-tests, create-test-deployments]
     strategy:
       matrix:
         deployment_id:
@@ -377,6 +462,13 @@ jobs:
       - name: Set CLI context
         run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }}
 
+      - name: Mock git commands for DAG Deploy
+        run: |
+
+          # set to no deploy git mock, to simulate no interested file change scenario
+          mv e2e-setup/mocks/no-deploy-git.sh /usr/local/bin/git
+          chmod +x /usr/local/bin/git
+
       - name: Get Deployment Info Before Test
         id: get-deployment-before
         if: ${{ matrix.deploy_type != 'dbt' }}

diff --git a/README.md b/README.md
@@ -49,7 +49,7 @@ The following table lists the configuration options for the Deploy to Astro acti
 | Name | Default | Description |
 | ---|---|--- |
 | `action` | `deploy` | Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either `deploy`, `create-deployment-preview`, `delete-deployment-preview` or `deploy-deployment-preview`. If using `deploy` or `deploy-deployment-preview` one should also specify `deploy-type`. |
-| `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs or DBT project. Possible options are `infer`, `dags-only`, `image-and-dags` or `dbt`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. |
+| `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs or DBT project. Possible options are `infer`, `dags-only`, `image-and-dags` or `dbt`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. For description on each deploy type checkout [deploy type details](https://github.com/astronomer/deploy-action#deploy-type-details) |
 | `deployment-id` | `false` | Specifies the id of the deployment you to make a preview from or are deploying too. |
 | `deployment-name` | `false` | Specifies The name of the deployment you want to make preview from or are deploying too. Cannot be used with `deployment-id` |
 | `description` |  | Configure a description for a deploy to Astro. Description will be visible in the Deploy History tab. |
@@ -76,6 +76,27 @@ The following table lists the outputs for the Deploy to Astro action.
 | ---|--- |
 | `preview-id` | The ID of the created deployment preview. Only works when action=create-deployment-preview. |
 
+## Deploy Type Details
+
+The following section describe each of the deploy type input value in detail to avoid any confusion:
+1. `infer`: In this mode, deploy-action would run through all the file changes:
+  - if there are no file changes in the configured root-folder then it skips deploy
+  - if there are changes only in `dags/` folder, then it will do a dags deploy
+  - otherwise it would do a complete image deploy
+
+2. `image-and-dags`: In this mode, deploy-action would run through all the file changes:
+  - if there are no file changes in the configured root-folder then it skips deploy
+  - otherwise it would do a complete image deploy
+
+3. `dags-only`: In this mode, deploy-action would run through all the file changes:
+  - if there are no file changes in the configured root-folder then it skips deploy
+  - if all the file changes are in folders except `dags` folder then it skips deploy
+  - otherwise it would do a dag only deploy
+
+4. `dbt`: In this mode, deploy-action would run through all the file changes:
+  - if there are no file changes in the configured root-folder then it skips deploy
+  - otherwise it would do a dbt deploy
+
 
 ## Workflow file examples
 

diff --git a/action.yaml b/action.yaml
@@ -300,7 +300,11 @@ runs:
       run: |
 
         echo ::group::Get DBT Deploy Options
-        cd ${{ inputs.root-folder }}
+
+        if [[ "${{ inputs.root-folder }}" != "" ]]; then
+          cd ${{ inputs.root-folder }}
+        fi
+
         branch=$(echo "${GITHUB_REF#refs/heads/}")
         echo "Branch pushed to: $branch"
         git fetch origin $branch
@@ -339,27 +343,39 @@ runs:
 
         # infer based on files changed to deploy only dags or image and dags
         echo ::group::Get Deploy Type
-        cd ${{ inputs.root-folder }}
+
+        if [[ "${{ inputs.root-folder }}" != "" ]]; then
+          cd ${{ inputs.root-folder }}
+        fi
+
         branch=$(echo "${GITHUB_REF#refs/heads/}")
         echo "Branch pushed to: $branch"
         git fetch origin $branch
         files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }})
         echo "files changed: $files"
-        DAGS_ONLY_DEPLOY=false
 
+        DAGS_ONLY_DEPLOY=false
         SKIP_IMAGE_OR_DAGS_DEPLOY=true
 
-        # This for loop checks for following cases:
-        # 1. If no file is part of the input root folder, then it skips deploy
-        # 2. If any file is not part of the dags folder, then it triggers a full image build
-        # 3. If all files are part of the dags folder and input root folder, then it triggers a DAG-only deploy
         for file in $files; do
           if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then
             echo $file is part of the input root folder
             SKIP_IMAGE_OR_DAGS_DEPLOY=false
-            if [[ $file == *"dags/"* && ${{ inputs.deploy-type }} != 'image-and-dags' ]]; then
-              echo $file is part of dags folder, triggering a DAG-only deploy
-              DAGS_ONLY_DEPLOY=true
+            if [[ ${{ inputs.deploy-type }} == 'infer' ]]; then
+              if [[ $file == *"dags/"* ]]; then
+                echo $file is part of dags folder
+                DAGS_ONLY_DEPLOY=true
+              else
+                DAGS_ONLY_DEPLOY=false
+                break
+              fi
+            elif [[ ${{ inputs.deploy-type }} == 'dags-only' ]]; then
+              if [[ $file == *"dags/"* ]]; then
+                echo $file is part of dags folder
+                DAGS_ONLY_DEPLOY=true
+              fi
+            elif [[ ${{ inputs.deploy-type }} == 'image-and-dags' ]]; then
+              DAGS_ONLY_DEPLOY=false
               break
             fi
           fi
@@ -439,7 +455,11 @@ runs:
 
         echo ::group::DAG Deploy to Astro
         # Deploy only dags
-        cd ${{ inputs.root-folder }}
+
+        if [[ "${{ inputs.root-folder }}" != "" ]]; then
+          cd ${{ inputs.root-folder }}
+        fi
+
         astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --dags ${{steps.deploy-options.outputs.OPTIONS}}
         echo ::endgroup::
       shell: bash
@@ -449,7 +469,11 @@ runs:
       run: |
         echo ::group::Image and DAG Deploy to Astro
         # Deploy image and DAGs
-        cd ${{ inputs.root-folder }}
+
+        if [[ "${{ inputs.root-folder }}" != "" ]]; then
+          cd ${{ inputs.root-folder }}
+        fi
+
         astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.deploy-options.outputs.OPTIONS}}
         echo ::endgroup::
       shell: bash
@@ -458,7 +482,11 @@ runs:
       run: |
 
         echo ::group::DBT Deploy to Astro
-        cd ${{ inputs.root-folder }}
+
+        if [[ "${{ inputs.root-folder }}" != "" ]]; then
+          cd ${{ inputs.root-folder }}
+        fi
+
         astro dbt deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.dbt-deploy-options.outputs.DBT_OPTIONS}}
         echo ::endgroup::
       shell: bash

diff --git a/e2e-setup/mocks/astro-deploy.sh b/e2e-setup/mocks/astro-deploy.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# pre-req for this mock script would be to have the actual astro cli installed at /usr/local/bin/astro-original
+
+if [ "$1" = "deploy" ]; then
+    # Change directory to 'e2e-setup/astro-project' and then call original `astro deploy`
+    # so that we could simulate the default behavior without needing to have the astro project in base folder
+    echo "cd into astro project" && cd e2e-setup/astro-project && /usr/local/bin/astro-original "$@"
+else
+    # If it's not a `deploy` command, run the original `astro`
+    /usr/local/bin/astro-original "$@"
+fi
diff --git a/e2e-setup/mocks/no-deploy-git.sh b/e2e-setup/mocks/no-deploy-git.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# hack to mock git commands as part of action.yaml so that we could simulate no deploy scenario without making any additional commits
+
+# Check if the script was invoked with "git diff"
+if [[ "$1" == "diff" ]]; then
+   echo "README.md"
+elif [[ "$1" == "fetch" ]]; then
+  echo "Handling git fetch, doing nothing"
+else
+  echo "Error: git mock script isn't configured to handle $1" >&2
+  exit 1
+fi