CI: Add GitHub workflows, including testing, linting, release drafting, and documentation generation

.github/dependabot.yml

@@ -0,0 +1,17 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "github-actions"
+    # Workflow files stored in the default location of `.github/workflows`. (You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.)
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/release-drafter.yml

@@ -0,0 +1,63 @@
+name-template: 'v$RESOLVED_VERSION'
+tag-template: 'v$RESOLVED_VERSION'
+categories:
+  - title: 'New Features ✨'
+    labels:
+      - 'feature'
+      - 'enhancement'
+  - title: 'Bug Fixes 🐛'
+    labels:
+      - 'fix'
+      - 'bugfix'
+      - 'bug'
+  - title: 'Under the Hood ⚙️'
+    labels:
+      - 'chore'
+      - 'ci'
+      - 'refactor'
+  - title: 'Documentation 📖'
+    label: 'docs'
+change-template: '- $SUBJECT (#$NUMBER) - **_Thanks, @$AUTHOR_**!'
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+version-resolver:
+  major:
+    labels:
+      - 'major'
+  minor:
+    labels:
+      - 'minor'
+  patch:
+    labels:
+      - 'patch'
+  default: patch
+template: |
+  ## Changes
+
+  $CHANGES
+autolabeler:
+  - label: 'chore'
+    title:
+      - '/^chore(\(.*\))?\:/i'
+  - label: 'ci'
+    title:
+      - '/^ci(\(.*\))?\:/i'
+  - label: 'bug'
+    title:
+      - '/^fix(\(.*\))?\:/i'
+  - label: 'enhancement'
+    title:
+      - '/^feat(\(.*\))?/i'
+  - label: 'docs'
+    title:
+      - '/^docs(\(.*\))?\:/i'
+  - label: 'security'
+    title:
+      - '/^security(\(.*\))?\:/i'
+      - '/^fix(\(security\))?\:/i'
+  - label: 'dependencies'
+    title:
+      - '/^chore\(deps\)\:/i'
+      - '/^build\(deps\)\:/i'
+  - label: 'breaking'
+    title:
+      - '/!:\s*$/i'

.github/workflows/connector-tests.yml

@@ -0,0 +1,122 @@
+name: Connectors Tests
+
+concurrency:
+  # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
+  #
+  # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests
+  #  ensuring that only one run per pull request is active at a time.
+  #
+  # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches.
+  # This allows us to run multiple workflow dispatches in parallel.
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+jobs:
+  cdk_changes:
+    name: Get Changes
+    runs-on: ubuntu-latest
+    permissions:
+      statuses: write
+      pull-requests: read
+    steps:
+      - name: Checkout Airbyte
+        if: github.event_name != 'pull_request'
+        uses: actions/checkout@v4
+      - id: changes
+        uses: dorny/[email protected]
+        with:
+          filters: |
+            src:
+              - 'airbyte_cdk/**'
+              - 'bin/**'
+              - 'poetry.lock'
+              - 'pyproject.toml'
+            file-based:
+              - 'airbyte_cdk/sources/file_based/**'
+            vector-db-based:
+              - 'airbyte_cdk/destinations/vector_db_based/**'
+            sql:
+              - 'airbyte_cdk/sql/**'
+    outputs:
+      # Source code modified:
+      src: ${{ steps.changes.outputs.src }}
+      # Extras modified:
+      file-based: ${{ steps.changes.outputs.file-based }}
+      vector-db-based: ${{ steps.changes.outputs.vector-db-based }}
+      sql: ${{ steps.changes.outputs.sql }}
+
+
+    #   # The Connector CI Tests is a status check emitted by airbyte-ci
+    #   # We make it pass once we have determined that there are no changes to the connectors
+    #   - name: "Skip Connectors CI tests"
+    #     if: steps.changes.outputs.src != 'true' && github.event_name == 'pull_request'
+    #     run: |
+    #       curl --request POST \
+    #       --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
+    #       --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+    #       --header 'content-type: application/json' \
+    #       --data '{
+    #         "state": "success",
+    #         "context": "CDK Changes - Connectors Tests",
+    #         "target_url": "${{ github.event.workflow_run.html_url }}"
+    #         }' \
+
+  connectors_ci:
+    needs: cdk_changes
+    # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR
+    # Forked PRs are handled by the community_ci.yml workflow
+    # If the condition is not met the job will be skipped (it will not fail)
+    # runs-on: connector-test-large
+    runs-on: ubuntu-latest
+    timeout-minutes: 360 # 6 hours
+    strategy:
+      fail-fast: true  # Save resources by aborting if one connector fails
+      matrix:
+        include:
+          - connector: source-shopify
+            cdk_extra: n/a
+          - connector: source-zendesk-support
+            cdk_extra: n/a
+          - connector: source-s3
+            cdk_extra: file-based
+          - connector: destination-pinecone
+            cdk_extra: vector-db-based
+          - connector: destination-motherduck
+            cdk_extra: sql
+    if: >
+      ( github.event_name == 'pull_request' && needs.cdk_changes.outputs.src == 'true' && github.event.pull_request.head.repo.fork != true
+      ) || github.event_name == 'workflow_dispatch'
+    name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
+    steps:
+      - name: Abort if extra not changed (${{matrix.cdk_extra}})
+        id: no_changes
+        if: ${{ matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }}
+        run: |
+            echo "Aborting job as specified extra not changed: ${{matrix.cdk_extra}} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}"
+            echo "status=cancelled" >> $GITHUB_OUTPUT
+            exit 1
+        continue-on-error: true
+      # Get the monorepo so we can test the connectors
+      - name: Checkout Airbyte Monorepo
+        uses: actions/checkout@v4
+        if: steps.no_changes.outcome != 'failure'
+        with:
+          repository: airbytehq/airbyte
+          ref: master
+      - name: Test Connector
+        if: steps.no_changes.outcome != 'failure'
+        timeout-minutes: 90
+        env:
+          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
+        run: |
+          make tools.airbyte-ci-binary.install
+          airbyte-ci connectors \
+          --name ${{matrix.connector}} \
+          test
+          --global-status-check-context='Connectors Test: ${{matrix.connector}}'"

.github/workflows/fix-pr-command.yml

@@ -0,0 +1,171 @@
+name: On-Demand PR Auto-Fix
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr:
+        description: 'PR Number'
+        type: string
+        required: true
+      comment-id:
+        description: 'Comment ID (Optional)'
+        type: string
+        required: false
+
+jobs:
+  # This is copied from the `python_pytest.yml` file.
+  # Only the first two steps of the job are different, and they check out the PR's branch.
+  pr-fix-on-demand:
+    name: On-Demand PR Fix
+    # Don't run on forks. Run on pushes to main, and on PRs that are not from forks.
+    strategy:
+      matrix:
+        python-version: [
+          '3.10',
+        ]
+        os: [
+          Ubuntu,
+        ]
+      fail-fast: false
+
+    runs-on: "${{ matrix.os }}-latest"
+    steps:
+
+    # Custom steps to fetch the PR and checkout the code:
+    - name: Checkout Airbyte
+      uses: actions/checkout@v4
+      with:
+        # Important that this is set so that CI checks are triggered again
+        # Without this we would be forever waiting on required checks to pass
+        token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+
+    - name: Checkout PR (${{ github.event.inputs.pr }})
+      uses: dawidd6/action-checkout-pr@v1
+      with:
+        pr: ${{ github.event.inputs.pr }}
+
+    - name: Get PR info
+      id: pr-info
+      run: |
+        PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }})
+        echo "repo=$(echo "$PR_JSON" | jq -r .head.repo.full_name)" >> $GITHUB_OUTPUT
+        echo "branch=$(echo "$PR_JSON" | jq -r .head.ref)" >> $GITHUB_OUTPUT
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      shell: bash
+
+    - name: Create URL to the run output
+      id: vars
+      run: echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
+
+    - name: Append comment with job run link
+      id: first-comment-action
+      uses: peter-evans/create-or-update-comment@v4
+      with:
+        comment-id: ${{ github.event.inputs.comment-id }}
+        issue-number: ${{ github.event.inputs.pr }}
+        body: |
+          > **Auto-Fix Job Info**
+          >
+          > This job attempts to auto-fix any linting or formating issues. If any fixes are made,
+          > those changes will be automatically committed and pushed back to the PR.
+          >
+          > Note: This job can only be run by maintainers. On PRs from forks, this command requires
+          > that the PR author has enabled the `Allow edits from maintainers` option.
+
+          > PR auto-fix job started... [Check job output.][1]
+
+          [1]: ${{ steps.vars.outputs.run-url }}
+
+    - name: Set up Poetry
+      uses: Gr1N/setup-poetry@v9
+      with:
+        poetry-version: "1.7.1"
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'poetry'
+    - name: Install dependencies
+      run: poetry install --all-extras
+
+    # Fix any lint or format issues
+
+    - name: Auto-Fix Ruff Lint Issues
+      run: poetry run ruff check --fix . || true
+    - name: Auto-Fix Ruff Format Issues
+      run: poetry run ruff format . || true
+
+    # Check for changes in git
+
+    - name: Check for changes
+      id: git-diff
+      run: |
+        git diff --quiet && echo "No changes to commit" || echo "changes=true" >> $GITHUB_OUTPUT
+      shell: bash
+
+    # Commit changes (if any)
+
+    - name: Commit changes
+      if: steps.git-diff.outputs.changes == 'true'
+      run: |
+        git config --global user.name "octavia-squidington-iii"
+        git config --global user.email "[email protected]"
+        git add .
+        git commit -m "Auto-fix lint and format issues"
+
+    # Fix any further 'unsafe' lint issues in a separate commit
+
+    - name: Auto-Fix Ruff Lint Issues (Unsafe)
+      run: poetry run ruff check --fix --unsafe-fixes . || true
+    - name: Auto-Fix Ruff Format Issues
+      run: poetry run ruff format . || true
+
+    # Check for changes in git (2nd time, for 'unsafe' lint fixes)
+
+    - name: Check for changes ('unsafe' fixes)
+      id: git-diff-2
+      run: |
+        git diff --quiet && echo "No changes to commit" || echo "changes=true" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Commit 'unsafe' lint fixes
+      if: steps.git-diff-2.outputs.changes == 'true'
+      run: |
+        git config --global user.name "octavia-squidington-iii"
+        git config --global user.email "[email protected]"
+        git add .
+        git commit -m "Auto-fix lint issues (unsafe)"
+
+    - name: Push changes to '(${{ steps.pr-info.outputs.repo }})'
+      if: steps.git-diff.outputs.changes == 'true' || steps.git-diff-2.outputs.changes == 'true'
+      run: |
+        git remote add contributor https://github.com/${{ steps.pr-info.outputs.repo }}.git
+        git push contributor HEAD:${{ steps.pr-info.outputs.branch }}
+
+    - name: Append success comment
+      uses: peter-evans/create-or-update-comment@v4
+      if: steps.git-diff.outputs.changes == 'true' || steps.git-diff-2.outputs.changes == 'true'
+      with:
+        comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
+        reactions: hooray
+        body: |
+          > ✅ Changes applied successfully.
+
+    - name: Append success comment (no-op)
+      uses: peter-evans/create-or-update-comment@v4
+      if: steps.git-diff.outputs.changes != 'true' && steps.git-diff-2.outputs.changes != 'true'
+      with:
+        comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
+        reactions: "+1"
+        body: |
+          > 🟦 Job completed successfully (no changes).
+
+    - name: Append failure comment
+      uses: peter-evans/create-or-update-comment@v4
+      if: failure()
+      with:
+        comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
+        reactions: confused
+        body: |
+          > ❌ Job failed.