From 6e70894117cba69b9e916950a02031e59604bb24 Mon Sep 17 00:00:00 2001 From: Neel Dalsania Date: Mon, 30 Sep 2024 18:31:36 +0530 Subject: [PATCH] Fix root folder reference (#84) --- .github/workflows/tests.yaml | 94 +++++++++++++++++++++++++++++++- README.md | 23 +++++++- action.yaml | 54 +++++++++++++----- e2e-setup/mocks/astro-deploy.sh | 12 ++++ e2e-setup/mocks/no-deploy-git.sh | 13 +++++ 5 files changed, 181 insertions(+), 15 deletions(-) create mode 100644 e2e-setup/mocks/astro-deploy.sh create mode 100755 e2e-setup/mocks/no-deploy-git.sh diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e7a9731..58b9ff2 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -100,11 +100,96 @@ jobs: DEPLOYMENT_ID=$(astro deployment create --deployment-file e2e-setup/deployment-templates/${{ matrix.deployment }} | yq e '.deployment.metadata.deployment_id' -) echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT + default-deploy-tests: + name: Default Deploy Test + runs-on: ubuntu-latest + needs: [create-test-deployments] + strategy: + max-parallel: 1 + matrix: + deployment_id: + ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"] + deploy_type: [dags, image-and-dags] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + + - name: Install dependencies + run: | + + sudo apt-get install jq + + # we need to pre-install the CLI to set the context + curl -sSL https://install.astronomer.io | sudo bash -s + + - name: Mock git commands + run: | + + # if deploy_type is dags, then use dags-deploy-git.sh as mock otherwise use image-deploy-git.sh as mock + if [[ "${{ matrix.deploy_type }}" == "dags" ]]; then + mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git + else + mv e2e-setup/mocks/image-deploy-git.sh /usr/local/bin/git + fi + chmod +x /usr/local/bin/git + cp /usr/local/bin/astro /usr/local/bin/astro-original + mv e2e-setup/mocks/astro-deploy.sh /usr/local/bin/astro + chmod +x /usr/local/bin/astro + + - name: Set CLI context + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Get Deployment Info Before Test + id: get-deployment-before + uses: ./.github/workflows/e2e/get_deployment_info + with: + deployment_id: ${{ matrix.deployment_id }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Run Deploy Action + uses: ./ + with: + deployment-id: ${{ matrix.deployment_id }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + + - name: Get Deployment Info After Test + id: get-deployment-after + uses: ./.github/workflows/e2e/get_deployment_info + with: + deployment_id: ${{ matrix.deployment_id }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Validate Deploy Action + uses: ./.github/workflows/e2e/validate_deployment + with: + is_dag_only_deploy: ${{ matrix.deploy_type == 'dags' }} + dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} + image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} + dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} + image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} + # DAG Deploy test would test the DAG only deploy functionality in deploy action dag-deploy-test: name: DAG Deploy Test runs-on: ubuntu-latest - needs: [create-test-deployments] + needs: [default-deploy-tests, create-test-deployments] strategy: matrix: deployment_id: @@ -377,6 +462,13 @@ jobs: - name: Set CLI context run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + - name: Mock git commands for DAG Deploy + run: | + + # set to no deploy git mock, to simulate no interested file change scenario + mv e2e-setup/mocks/no-deploy-git.sh /usr/local/bin/git + chmod +x /usr/local/bin/git + - name: Get Deployment Info Before Test id: get-deployment-before if: ${{ matrix.deploy_type != 'dbt' }} diff --git a/README.md b/README.md index 626763e..81985ad 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The following table lists the configuration options for the Deploy to Astro acti | Name | Default | Description | | ---|---|--- | | `action` | `deploy` | Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either `deploy`, `create-deployment-preview`, `delete-deployment-preview` or `deploy-deployment-preview`. If using `deploy` or `deploy-deployment-preview` one should also specify `deploy-type`. | -| `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs or DBT project. Possible options are `infer`, `dags-only`, `image-and-dags` or `dbt`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. | +| `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs or DBT project. Possible options are `infer`, `dags-only`, `image-and-dags` or `dbt`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. For description on each deploy type checkout [deploy type details](https://github.com/astronomer/deploy-action#deploy-type-details) | | `deployment-id` | `false` | Specifies the id of the deployment you to make a preview from or are deploying too. | | `deployment-name` | `false` | Specifies The name of the deployment you want to make preview from or are deploying too. Cannot be used with `deployment-id` | | `description` | | Configure a description for a deploy to Astro. Description will be visible in the Deploy History tab. | @@ -76,6 +76,27 @@ The following table lists the outputs for the Deploy to Astro action. | ---|--- | | `preview-id` | The ID of the created deployment preview. Only works when action=create-deployment-preview. | +## Deploy Type Details + +The following section describe each of the deploy type input value in detail to avoid any confusion: +1. `infer`: In this mode, deploy-action would run through all the file changes: + - if there are no file changes in the configured root-folder then it skips deploy + - if there are changes only in `dags/` folder, then it will do a dags deploy + - otherwise it would do a complete image deploy + +2. `image-and-dags`: In this mode, deploy-action would run through all the file changes: + - if there are no file changes in the configured root-folder then it skips deploy + - otherwise it would do a complete image deploy + +3. `dags-only`: In this mode, deploy-action would run through all the file changes: + - if there are no file changes in the configured root-folder then it skips deploy + - if all the file changes are in folders except `dags` folder then it skips deploy + - otherwise it would do a dag only deploy + +4. `dbt`: In this mode, deploy-action would run through all the file changes: + - if there are no file changes in the configured root-folder then it skips deploy + - otherwise it would do a dbt deploy + ## Workflow file examples diff --git a/action.yaml b/action.yaml index fab3e75..a86675e 100644 --- a/action.yaml +++ b/action.yaml @@ -300,7 +300,11 @@ runs: run: | echo ::group::Get DBT Deploy Options - cd ${{ inputs.root-folder }} + + if [[ "${{ inputs.root-folder }}" != "" ]]; then + cd ${{ inputs.root-folder }} + fi + branch=$(echo "${GITHUB_REF#refs/heads/}") echo "Branch pushed to: $branch" git fetch origin $branch @@ -339,27 +343,39 @@ runs: # infer based on files changed to deploy only dags or image and dags echo ::group::Get Deploy Type - cd ${{ inputs.root-folder }} + + if [[ "${{ inputs.root-folder }}" != "" ]]; then + cd ${{ inputs.root-folder }} + fi + branch=$(echo "${GITHUB_REF#refs/heads/}") echo "Branch pushed to: $branch" git fetch origin $branch files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }}) echo "files changed: $files" - DAGS_ONLY_DEPLOY=false + DAGS_ONLY_DEPLOY=false SKIP_IMAGE_OR_DAGS_DEPLOY=true - # This for loop checks for following cases: - # 1. If no file is part of the input root folder, then it skips deploy - # 2. If any file is not part of the dags folder, then it triggers a full image build - # 3. If all files are part of the dags folder and input root folder, then it triggers a DAG-only deploy for file in $files; do if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then echo $file is part of the input root folder SKIP_IMAGE_OR_DAGS_DEPLOY=false - if [[ $file == *"dags/"* && ${{ inputs.deploy-type }} != 'image-and-dags' ]]; then - echo $file is part of dags folder, triggering a DAG-only deploy - DAGS_ONLY_DEPLOY=true + if [[ ${{ inputs.deploy-type }} == 'infer' ]]; then + if [[ $file == *"dags/"* ]]; then + echo $file is part of dags folder + DAGS_ONLY_DEPLOY=true + else + DAGS_ONLY_DEPLOY=false + break + fi + elif [[ ${{ inputs.deploy-type }} == 'dags-only' ]]; then + if [[ $file == *"dags/"* ]]; then + echo $file is part of dags folder + DAGS_ONLY_DEPLOY=true + fi + elif [[ ${{ inputs.deploy-type }} == 'image-and-dags' ]]; then + DAGS_ONLY_DEPLOY=false break fi fi @@ -439,7 +455,11 @@ runs: echo ::group::DAG Deploy to Astro # Deploy only dags - cd ${{ inputs.root-folder }} + + if [[ "${{ inputs.root-folder }}" != "" ]]; then + cd ${{ inputs.root-folder }} + fi + astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --dags ${{steps.deploy-options.outputs.OPTIONS}} echo ::endgroup:: shell: bash @@ -449,7 +469,11 @@ runs: run: | echo ::group::Image and DAG Deploy to Astro # Deploy image and DAGs - cd ${{ inputs.root-folder }} + + if [[ "${{ inputs.root-folder }}" != "" ]]; then + cd ${{ inputs.root-folder }} + fi + astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.deploy-options.outputs.OPTIONS}} echo ::endgroup:: shell: bash @@ -458,7 +482,11 @@ runs: run: | echo ::group::DBT Deploy to Astro - cd ${{ inputs.root-folder }} + + if [[ "${{ inputs.root-folder }}" != "" ]]; then + cd ${{ inputs.root-folder }} + fi + astro dbt deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.dbt-deploy-options.outputs.DBT_OPTIONS}} echo ::endgroup:: shell: bash diff --git a/e2e-setup/mocks/astro-deploy.sh b/e2e-setup/mocks/astro-deploy.sh new file mode 100644 index 0000000..e8357b8 --- /dev/null +++ b/e2e-setup/mocks/astro-deploy.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# pre-req for this mock script would be to have the actual astro cli installed at /usr/local/bin/astro-original + +if [ "$1" = "deploy" ]; then + # Change directory to 'e2e-setup/astro-project' and then call original `astro deploy` + # so that we could simulate the default behavior without needing to have the astro project in base folder + echo "cd into astro project" && cd e2e-setup/astro-project && /usr/local/bin/astro-original "$@" +else + # If it's not a `deploy` command, run the original `astro` + /usr/local/bin/astro-original "$@" +fi diff --git a/e2e-setup/mocks/no-deploy-git.sh b/e2e-setup/mocks/no-deploy-git.sh new file mode 100755 index 0000000..f826c93 --- /dev/null +++ b/e2e-setup/mocks/no-deploy-git.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# hack to mock git commands as part of action.yaml so that we could simulate no deploy scenario without making any additional commits + +# Check if the script was invoked with "git diff" +if [[ "$1" == "diff" ]]; then + echo "README.md" +elif [[ "$1" == "fetch" ]]; then + echo "Handling git fetch, doing nothing" +else + echo "Error: git mock script isn't configured to handle $1" >&2 + exit 1 +fi