diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..7ef56f4a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + + - package-ecosystem: "github-actions" + # Workflow files stored in the default location of `.github/workflows`. (You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.) + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 00000000..0f6a2bc0 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,46 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +categories: + - title: 'New Features ✨' + labels: + - 'feature' + - 'enhancement' + - title: 'Bug Fixes 🐛' + labels: + - 'fix' + - 'bugfix' + - 'bug' + - title: 'Under the Hood ⚙️' + labels: + - 'chore' + - 'ci' + - 'refactor' + - title: 'Documentation 📖' + label: 'docs' +change-template: '- $TITLE @$AUTHOR (#$NUMBER)' +change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks. +version-resolver: + major: + labels: + - 'major' + minor: + labels: + - 'minor' + patch: + labels: + - 'patch' + default: patch +template: | + ## Changes + + $CHANGES +autolabeler: + - label: 'chore' + title: + - '/chore\:/i' + - label: 'bug' + title: + - '/fix\:/i' + - label: 'enhancement' + title: + - '/feature/i' diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml new file mode 100644 index 00000000..54580b4c --- /dev/null +++ b/.github/workflows/connector-tests.yml @@ -0,0 +1,128 @@ +name: Connectors Tests + +concurrency: + # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. + # + # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests + # ensuring that only one run per pull request is active at a time. + # + # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches. + # This allows us to run multiple workflow dispatches in parallel. + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + types: + - opened + - synchronize +jobs: + cdk_changes: + name: Get Changes + runs-on: ubuntu-latest + permissions: + statuses: write + pull-requests: read + steps: + - name: Checkout Airbyte + if: github.event_name != 'pull_request' + uses: actions/checkout@v4 + - id: changes + uses: dorny/paths-filter@v3.0.2 + with: + filters: | + src: + - 'airbyte_cdk/**' + - 'bin/**' + - 'poetry.lock' + - 'pyproject.toml' + file-based: + - 'airbyte_cdk/sources/file_based/**' + vector-db-based: + - 'airbyte_cdk/destinations/vector_db_based/**' + sql: + - 'airbyte_cdk/sql/**' + outputs: + # Source code modified: + src: ${{ steps.changes.outputs.src }} + # Extras modified: + file-based: ${{ steps.changes.outputs.file-based }} + vector-db-based: ${{ steps.changes.outputs.vector-db-based }} + sql: ${{ steps.changes.outputs.sql }} + + + # # The Connector CI Tests is a status check emitted by airbyte-ci + # # We make it pass once we have determined that there are no changes to the connectors + # - name: "Skip Connectors CI tests" + # if: steps.changes.outputs.src != 'true' && github.event_name == 'pull_request' + # run: | + # curl --request POST \ + # --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \ + # --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ + # --header 'content-type: application/json' \ + # --data '{ + # "state": "success", + # "context": "CDK Changes - Connectors Tests", + # "target_url": "${{ github.event.workflow_run.html_url }}" + # }' \ + + connectors_ci: + needs: cdk_changes + # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR + # Forked PRs are handled by the community_ci.yml workflow + # If the condition is not met the job will be skipped (it will not fail) + # runs-on: connector-test-large + runs-on: ubuntu-latest + timeout-minutes: 360 # 6 hours + strategy: + fail-fast: true # Save resources by aborting if one connector fails + matrix: + include: + - connector: source-shopify + cdk_extra: n/a + - connector: source-zendesk-support + cdk_extra: n/a + - connector: source-s3 + cdk_extra: file-based + - connector: destination-pinecone + cdk_extra: vector-db-based + - connector: destination-motherduck + cdk_extra: sql + if: > + ( github.event_name == 'pull_request' && needs.cdk_changes.outputs.src == 'true' && github.event.pull_request.head.repo.fork != true + ) || github.event_name == 'workflow_dispatch' + name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" + steps: + - name: Abort if extra not changed (${{matrix.cdk_extra}}) + id: no_changes + if: ${{ matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }} + run: | + echo "Aborting job as specified extra not changed: ${{matrix.cdk_extra}} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}" + echo "::set-output name=status::cancelled" + exit 1 + continue-on-error: true + # Get the monorepo so we can test the connectors + - name: Checkout Airbyte Monorepo + uses: actions/checkout@v4 + if: steps.no_changes.outcome != 'failure' + with: + repository: airbytehq/airbyte + ref: master + - name: Fetch last commit id from remote branch [PULL REQUESTS] + if: github.event_name == 'pull_request' && steps.no_changes.outcome != 'failure' + id: fetch_last_commit_id_pr + run: echo "commit_id=$(git ls-remote --heads origin refs/heads/${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT + - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH] + if: github.event_name == 'workflow_dispatch' && steps.no_changes.outcome != 'failure' + id: fetch_last_commit_id_wd + run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT + - name: Test Connector + if: steps.no_changes.outcome != 'failure' + timeout-minutes: 90 + run: | + make tools.airbyte-ci-binary.install + airbyte-ci connectors \ + --name ${{matrix.connector}} \ + test + --global-status-check-context='Connectors Test: ${{matrix.connector}}'" diff --git a/.github/workflows/fix-pr-command.yml b/.github/workflows/fix-pr-command.yml new file mode 100644 index 00000000..6eebae8b --- /dev/null +++ b/.github/workflows/fix-pr-command.yml @@ -0,0 +1,174 @@ +name: On-Demand PR Auto-Fix + +on: + workflow_dispatch: + inputs: + pr: + description: 'PR Number' + type: string + required: true + comment-id: + description: 'Comment ID (Optional)' + type: string + required: false + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +jobs: + # This is copied from the `python_pytest.yml` file. + # Only the first two steps of the job are different, and they check out the PR's branch. + pr-fix-on-demand: + name: On-Demand PR Fix + # Don't run on forks. Run on pushes to main, and on PRs that are not from forks. + strategy: + matrix: + python-version: [ + '3.10', + ] + os: [ + Ubuntu, + ] + fail-fast: false + + runs-on: "${{ matrix.os }}-latest" + steps: + + # Custom steps to fetch the PR and checkout the code: + - name: Checkout Airbyte + uses: actions/checkout@v4 + with: + # Important that this is set so that CI checks are triggered again + # Without this we would be forever waiting on required checks to pass + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + + - name: Checkout PR (${{ github.event.inputs.pr }}) + uses: dawidd6/action-checkout-pr@v1 + with: + pr: ${{ github.event.inputs.pr }} + + - name: Get PR info + id: pr-info + run: | + PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }}) + echo "::set-output name=repo::$(echo "$PR_JSON" | jq -r .head.repo.full_name)" + echo "::set-output name=branch::$(echo "$PR_JSON" | jq -r .head.ref)" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + + - name: Create URL to the run output + id: vars + run: echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT + + - name: Append comment with job run link + id: first-comment-action + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ github.event.inputs.comment-id }} + issue-number: ${{ github.event.inputs.pr }} + body: | + > **Auto-Fix Job Info** + > + > This job attempts to auto-fix any linting or formating issues. If any fixes are made, + > those changes will be automatically committed and pushed back to the PR. + > + > Note: This job can only be run by maintainers. On PRs from forks, this command requires + > that the PR author has enabled the `Allow edits from maintainers` option. + + > PR auto-fix job started... [Check job output.][1] + + [1]: ${{ steps.vars.outputs.run-url }} + + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + - name: Install dependencies + run: poetry install --all-extras + + # Fix any lint or format issues + + - name: Auto-Fix Ruff Lint Issues + run: poetry run ruff check --fix . || true + - name: Auto-Fix Ruff Format Issues + run: poetry run ruff format . || true + + # Check for changes in git + + - name: Check for changes + id: git-diff + run: | + git diff --quiet && echo "No changes to commit" || echo "::set-output name=changes::true" + shell: bash + + # Commit changes (if any) + + - name: Commit changes + if: steps.git-diff.outputs.changes == 'true' + run: | + git config --global user.name "octavia-squidington-iii" + git config --global user.email "contact@airbyte.com" + git add . + git commit -m "Auto-fix lint and format issues" + + # Fix any further 'unsafe' lint issues in a separate commit + + - name: Auto-Fix Ruff Lint Issues (Unsafe) + run: poetry run ruff check --fix --unsafe-fixes . || true + - name: Auto-Fix Ruff Format Issues + run: poetry run ruff format . || true + + # Check for changes in git (2nd time, for 'unsafe' lint fixes) + + - name: Check for changes ('unsafe' fixes) + id: git-diff-2 + run: | + git diff --quiet && echo "No changes to commit" || echo "::set-output name=changes::true" + shell: bash + + - name: Commit 'unsafe' lint fixes + if: steps.git-diff-2.outputs.changes == 'true' + run: | + git config --global user.name "octavia-squidington-iii" + git config --global user.email "contact@airbyte.com" + git add . + git commit -m "Auto-fix lint issues (unsafe)" + + - name: Push changes to '(${{ steps.pr-info.outputs.repo }})' + if: steps.git-diff.outputs.changes == 'true' || steps.git-diff-2.outputs.changes == 'true' + run: | + git remote add contributor https://github.com/${{ steps.pr-info.outputs.repo }}.git + git push contributor HEAD:${{ steps.pr-info.outputs.branch }} + + - name: Append success comment + uses: peter-evans/create-or-update-comment@v4 + if: steps.git-diff.outputs.changes == 'true' || steps.git-diff-2.outputs.changes == 'true' + with: + comment-id: ${{ steps.first-comment-action.outputs.comment-id }} + reactions: hooray + body: | + > ✅ Changes applied successfully. + + - name: Append success comment (no-op) + uses: peter-evans/create-or-update-comment@v4 + if: steps.git-diff.outputs.changes != 'true' && steps.git-diff-2.outputs.changes != 'true' + with: + comment-id: ${{ steps.first-comment-action.outputs.comment-id }} + reactions: "+1" + body: | + > 🟦 Job completed successfully (no changes). + + - name: Append failure comment + uses: peter-evans/create-or-update-comment@v4 + if: failure() + with: + comment-id: ${{ steps.first-comment-action.outputs.comment-id }} + reactions: confused + body: | + > ❌ Job failed. diff --git a/.github/workflows/poetry-lock-command.yml b/.github/workflows/poetry-lock-command.yml new file mode 100644 index 00000000..a4a1145d --- /dev/null +++ b/.github/workflows/poetry-lock-command.yml @@ -0,0 +1,144 @@ +name: On-Demand Poetry Lock + +on: + workflow_dispatch: + inputs: + pr: + description: 'PR Number' + type: string + required: true + comment-id: + description: 'Comment ID (Optional)' + type: string + required: false + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +jobs: + poetry-lock-on-demand: + name: On-Demand Poetry Lock + strategy: + matrix: + python-version: [ + '3.10', + ] + os: [ + Ubuntu, + ] + fail-fast: false + + runs-on: "${{ matrix.os }}-latest" + steps: + + # Custom steps to fetch the PR and checkout the code: + - name: Checkout Airbyte + uses: actions/checkout@v4 + with: + # Important that this is set so that CI checks are triggered again + # Without this we would be forever waiting on required checks to pass + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + + - name: Checkout PR (${{ github.event.inputs.pr }}) + uses: dawidd6/action-checkout-pr@v1 + with: + pr: ${{ github.event.inputs.pr }} + + - name: Get PR info + id: pr-info + run: | + PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }}) + echo "::set-output name=repo::$(echo "$PR_JSON" | jq -r .head.repo.full_name)" + echo "::set-output name=branch::$(echo "$PR_JSON" | jq -r .head.ref)" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + + - name: Create URL to the run output + id: vars + run: echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT + + - name: Append comment with job run link + id: first-comment-action + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ github.event.inputs.comment-id }} + issue-number: ${{ github.event.inputs.pr }} + body: | + > **Poetry-Lock Job Info** + > + > This job attempts to re-lock dependencies using `poetry lock` command. If any changes + > are made, those changes will be automatically committed and pushed back to the PR. + > + > Note: This job can only be run by maintainers. On PRs from forks, this command requires + > that the PR author has enabled the `Allow edits from maintainers` option. + > + > `poetry lock` job started... [Check job output.][1] + + [1]: ${{ steps.vars.outputs.run-url }} + + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + + # Run `poetry lock` + + - name: Run `poetry lock` + run: poetry lock + + # Check for changes in git + + - name: Check for changes + id: git-diff + run: | + git diff --quiet && echo "No changes to commit" || echo "::set-output name=changes::true" + shell: bash + + # Commit changes (if any) + + - name: Commit changes + if: steps.git-diff.outputs.changes == 'true' + run: | + git config --global user.name "octavia-squidington-iii" + git config --global user.email "contact@airbyte.com" + git add . + git commit -m "Auto-commit `poetry lock` changes" + + - name: Push changes to '(${{ steps.pr-info.outputs.repo }})' + if: steps.git-diff.outputs.changes == 'true' + run: | + git remote add contributor https://github.com/${{ steps.pr-info.outputs.repo }}.git + git push contributor HEAD:${{ steps.pr-info.outputs.branch }} + + - name: Append success comment + uses: peter-evans/create-or-update-comment@v4 + if: steps.git-diff.outputs.changes == 'true' + with: + comment-id: ${{ steps.first-comment-action.outputs.comment-id }} + reactions: hooray + body: | + > ✅ `poetry lock` applied successfully. + + - name: Append success comment (no-op) + uses: peter-evans/create-or-update-comment@v4 + if: steps.git-diff.outputs.changes != 'true' && steps.git-diff-2.outputs.changes != 'true' + with: + comment-id: ${{ steps.first-comment-action.outputs.comment-id }} + reactions: "+1" + body: | + > 🟦 Job completed successfully (no changes). + + - name: Append failure comment + uses: peter-evans/create-or-update-comment@v4 + if: failure() + with: + comment-id: ${{ steps.first-comment-action.outputs.comment-id }} + reactions: confused + body: | + > ❌ Job failed. diff --git a/.github/workflows/pydoc_preview.yml b/.github/workflows/pydoc_preview.yml new file mode 100644 index 00000000..3ce7e4d9 --- /dev/null +++ b/.github/workflows/pydoc_preview.yml @@ -0,0 +1,40 @@ +name: Generate Docs + +on: + push: + branches: + - main + pull_request: {} + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +jobs: + preview_docs: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + + - name: Install dependencies + run: poetry install --all-extras + + - name: Generate documentation + run: | + poetry run poe docs-generate + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + # Upload entire repository + path: 'docs/generated' diff --git a/.github/workflows/pydoc_publish.yml b/.github/workflows/pydoc_publish.yml new file mode 100644 index 00000000..bd70e1c8 --- /dev/null +++ b/.github/workflows/pydoc_publish.yml @@ -0,0 +1,63 @@ +name: Publish Docs + +on: + push: + branches: + - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + publish_docs: + runs-on: ubuntu-latest + environment: + name: "github-pages" + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + - name: Setup Pages + uses: actions/configure-pages@v5 + + - name: Install dependencies + run: poetry install --all-extras + + - name: Generate documentation + run: | + poetry run poe docs-generate + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + # Upload entire repository + path: 'docs/generated' + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml new file mode 100644 index 00000000..65d98d31 --- /dev/null +++ b/.github/workflows/pypi_publish.yml @@ -0,0 +1,46 @@ +name: Build and/or Publish + +on: + push: + + workflow_dispatch: + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: hynek/build-and-inspect-python-package@v2 + + publish: + name: Publish to PyPI + runs-on: ubuntu-latest + needs: [build] + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + contents: write # Needed to upload artifacts to the release + environment: + name: PyPi + url: https://pypi.org/p/airbyte + if: startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/download-artifact@v4 + with: + name: Packages + path: dist + - name: Upload wheel to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: dist/*.whl + tag: ${{ github.ref }} + overwrite: true + file_glob: true + + - name: Publish + uses: pypa/gh-action-pypi-publish@v1.10.3 diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml new file mode 100644 index 00000000..5598076c --- /dev/null +++ b/.github/workflows/python_lint.yml @@ -0,0 +1,88 @@ +name: Linters + +on: + push: + branches: + - main + pull_request: {} + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +jobs: + ruff-lint-check: + name: Ruff Lint Check + runs-on: ubuntu-latest + steps: + # Common steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + - name: Install dependencies + run: poetry install --all-extras + + # Job-specifc step(s): + - name: Format code + run: poetry run ruff check . + + ruff-format-check: + name: Ruff Format Check + runs-on: ubuntu-latest + steps: + # Common steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + - name: Install dependencies + run: poetry install --all-extras + + # Job-specifc step(s): + - name: Check code format + run: poetry run ruff format --check . + + mypy-check: + name: MyPy Check + runs-on: ubuntu-latest + steps: + # Common steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + - name: Install dependencies + run: poetry install --all-extras + + # Job-specifc step(s): + + # For now, we run mypy only on modified files + - name: Get changed Python files + id: changed-py-files + uses: tj-actions/changed-files@v43 + with: + files: "airbyte_cdk/**/*.py" + - name: Run mypy on changed files + if: steps.changed-py-files.outputs.any_changed == 'true' + run: mypy ${{ steps.changed-py-files.outputs.all_changed_files }} --config-file mypy.ini --install-types --non-interactive diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml new file mode 100644 index 00000000..1eaac864 --- /dev/null +++ b/.github/workflows/python_pytest.yml @@ -0,0 +1,140 @@ +# This workflow will run pytest. +# +# There are two job sets which run in parallel: +# 1. `pytest-fast`: Run fast tests only, and fail fast so the dev knows asap if they broke something. +# 2. `pytest`: Run all tests, across multiple Python versions. +# +# Note that `pytest-fast` also skips tests that require credentials, allowing it to run on forks. +name: PyTest + +on: + push: + branches: + - main + pull_request: {} + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +jobs: + pytest-fast: + name: Pytest (Fast) + runs-on: ubuntu-latest + steps: + # Common steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + - name: Install dependencies + run: poetry install --all-extras + + - name: Run Pytest with Coverage (Fast Tests Only) + timeout-minutes: 60 + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + run: > + poetry run coverage run -m pytest + --durations=5 --exitfirst + -m "not slow and not requires_creds and not linting and not flaky" + + - name: Run Pytest with Coverage (Flaky Tests Only) + timeout-minutes: 60 + continue-on-error: true + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + run: > + poetry run coverage run -m pytest + --durations=5 --exitfirst + -m "flaky and not slow and not requires_creds" + + - name: Print Coverage Report + if: always() + run: poetry run coverage report + + - name: Create Coverage Artifacts + if: always() + run: | + poetry run coverage html -d htmlcov + poetry run coverage xml -o htmlcov/coverage.xml + + - name: Upload coverage to GitHub Artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: fasttest-coverage + path: htmlcov/ + + pytest: + name: Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }}) + # Don't run on forks. Run on pushes to main, and on PRs that are not from forks. + if: > + (github.event_name == 'push' && github.ref == 'refs/heads/main') || + (github.event.pull_request.head.repo.fork == false) + strategy: + matrix: + python-version: [ + '3.10', + '3.11', + #'3.12', # Currently blocked by Pendulum + ] + os: [ + Ubuntu, + # Windows, # For now, we don't include Windows in the test matrix. + ] + fail-fast: false + + runs-on: "${{ matrix.os }}-latest" + env: + # Enforce UTF-8 encoding so Windows runners don't fail inside the connector code. + # TODO: See if we can fully enforce this within PyAirbyte itself. + PYTHONIOENCODING: utf-8 + steps: + # Common steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + - name: Install dependencies + run: poetry install --all-extras + + # Job-specific step(s): + - name: Run Pytest + timeout-minutes: 60 + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + run: > + poetry run coverage run -m pytest + --durations=10 + -m "not linting and not super_slow and not flaky" + + - name: Print Coverage Report + if: always() + run: poetry run coverage report + + - name: Create Coverage Artifacts + if: always() + run: | + poetry run coverage html -d htmlcov + poetry run coverage xml -o htmlcov/coverage.xml + + - name: Upload coverage to GitHub Artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: py${{ matrix.python-version }}-${{ matrix.os }}-test-coverage + path: htmlcov/ diff --git a/.github/workflows/release_drafter.yml b/.github/workflows/release_drafter.yml new file mode 100644 index 00000000..ea236d5c --- /dev/null +++ b/.github/workflows/release_drafter.yml @@ -0,0 +1,27 @@ +name: Release Drafter + +on: + push: + branches: + - main + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +permissions: + contents: read + +jobs: + update_release_draft: + permissions: + contents: write + pull-requests: read + runs-on: ubuntu-latest + steps: + # Drafts the next Release notes as Pull Requests are merged into "main" + - uses: release-drafter/release-drafter@v6 + with: + config-name: release-drafter.yml + disable-autolabeler: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/semantic_pr_check.yml b/.github/workflows/semantic_pr_check.yml new file mode 100644 index 00000000..1cdb7868 --- /dev/null +++ b/.github/workflows/semantic_pr_check.yml @@ -0,0 +1,50 @@ +name: "Verify Semantic PR Title" + +on: + pull_request: + types: + - opened + - edited + - synchronize + - ready_for_review + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +permissions: + pull-requests: read + +jobs: + validate_pr_title: + name: Validate PR title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v5 + if: ${{ github.event.pull_request.draft == false }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # Configure which types are allowed (newline-delimited). + # See: https://github.com/commitizen/conventional-commit-types/blob/master/index.json + types: | + Fix + Feat + Docs + CI + Chore + Build + Test + + # # We don't use scopes as of now + # scopes: | + # core + # ui + # JIRA-\d+ + + # Require capitalization for the first letter of the subject. + subjectPattern: ^[A-Z].*$ + # The variables `subject` and `title` can be used within the message. + subjectPatternError: | + The subject "{subject}" found in the pull request title "{title}" + didn't match the configured pattern. Please ensure that the subject + start with an uppercase character. diff --git a/.github/workflows/slash_command_dispatch.yml b/.github/workflows/slash_command_dispatch.yml new file mode 100644 index 00000000..25c92ad5 --- /dev/null +++ b/.github/workflows/slash_command_dispatch.yml @@ -0,0 +1,42 @@ +name: Slash Command Dispatch + +on: + issue_comment: + types: [created] + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +jobs: + slashCommandDispatch: + # Only allow slash commands on pull request (not on issues) + if: ${{ github.event.issue.pull_request }} + runs-on: ubuntu-latest + steps: + + - name: Slash Command Dispatch + id: dispatch + uses: peter-evans/slash-command-dispatch@v4 + with: + repository: ${{ github.repository }} + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + dispatch-type: workflow + issue-type: pull-request + commands: | + fix-pr + test-pr + poetry-lock + static-args: | + pr=${{ github.event.issue.number }} + comment-id=${{ github.event.comment.id }} + + # Only run for users with 'write' permission on the main repository + permission: write + + - name: Edit comment with error message + if: steps.dispatch.outputs.error-message + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ github.event.comment.id }} + body: | + > Error: ${{ steps.dispatch.outputs.error-message }} diff --git a/.github/workflows/test-pr-command.yml b/.github/workflows/test-pr-command.yml new file mode 100644 index 00000000..2636f46d --- /dev/null +++ b/.github/workflows/test-pr-command.yml @@ -0,0 +1,163 @@ +name: On-Demand PR Test + +on: + workflow_dispatch: + inputs: + pr: + description: 'PR Number' + type: string + required: true + comment-id: + description: 'Comment ID (Optional)' + type: string + required: false + +env: + AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }} + +jobs: + start-workflow: + name: Append 'Starting' Comment + runs-on: ubuntu-latest + steps: + - name: Get PR JSON + id: pr-info + env: + GH_TOKEN: ${{ github.token }} + run: | + PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }}) + echo "$PR_JSON" > pr-info.json + echo "sha=$(cat pr-info.json | jq -r .head.sha)" >> $GITHUB_OUTPUT + echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT + - name: Upload PR details as artifact + uses: actions/upload-artifact@v4 + with: + name: pr-info + path: pr-info.json + - name: Append comment with job run link + id: first-comment-action + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ github.event.inputs.comment-id }} + issue-number: ${{ github.event.inputs.pr }} + body: | + + > PR test job started... [Check job output.][1] + + [1]: ${{ steps.pr-info.outputs.run-url }} + + # This is copied from the `python_pytest.yml` file. + # Only the first two steps of the job are different, and they check out the PR's branch. + pytest-on-demand: + name: On-Demand PR Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }}) + needs: [start-workflow] + strategy: + matrix: + python-version: [ + '3.10', + '3.11', + ] + os: [ + Ubuntu, + # Windows, # For now, we don't include Windows in the test matrix. + ] + fail-fast: false + runs-on: "${{ matrix.os }}-latest" + env: + # Enforce UTF-8 encoding so Windows runners don't fail inside the connector code. + # TODO: See if we can fully enforce this within PyAirbyte itself. + PYTHONIOENCODING: utf-8 + steps: + + # Custom steps to fetch the PR and checkout the code: + + - name: Download PR info + # This puts the `pr-info.json` file in the current directory. + # We need this to get the PR's SHA at the time of the workflow run. + uses: actions/download-artifact@v4 + with: + name: pr-info + + - name: Checkout PR + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Checkout PR (${{ github.event.inputs.pr }}) + uses: dawidd6/action-checkout-pr@v1 + with: + pr: ${{ github.event.inputs.pr }} + + # Same as the `python_pytest.yml` file: + + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'poetry' + - name: Install dependencies + run: poetry install --all-extras + + - name: Run Pytest + timeout-minutes: 60 + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + run: > + poetry run pytest + --verbose + -m "not super_slow and not flaky" + + - name: Run Pytest (Flaky Only) + continue-on-error: true + timeout-minutes: 60 + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + run: > + poetry run pytest + --verbose + -m "flaky and not super_slow" + + - name: Post CI Success to GitHub + run: | + curl --request POST \ + --url https://api.github.com/repos/${{ github.repository }}/statuses/$(cat pr-info.json | jq -r .head.sha) \ + --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ + --header 'content-type: application/json' \ + --data '{ + "state": "success", + "context": "Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }})", + "target_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", + }' \ + + log-success-comment: + name: Append 'Success' Comment + needs: [pytest-on-demand] + runs-on: ubuntu-latest + steps: + - name: Append success comment + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.inputs.pr }} + comment-id: ${{ github.event.inputs.comment-id }} + reactions: hooray + body: | + > ✅ Tests passed. + + log-failure-comment: + name: Append 'Failure' Comment + # This job will only run if the workflow fails + needs: [pytest-on-demand, start-workflow] + if: always() && needs.pytest-on-demand.result == 'failure' + runs-on: ubuntu-latest + steps: + - name: Append failure comment + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.inputs.pr }} + comment-id: ${{ github.event.inputs.comment-id }} + reactions: confused + body: | + > ❌ Tests failed.