From 5ea6e6e7347e7f88512986bc39de3f66858d01ac Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Thu, 19 Sep 2024 10:54:31 -0400 Subject: [PATCH 1/3] chore: split metadata extract across several workflows (#1849) --- .../workflows/metadata-extract-airbyte.yml | 89 ++++++++ .../workflows/metadata-extract-extractors.yml | 190 ++++++++++++++++++ .../workflows/metadata-extract-loaders.yml | 122 +++++++++++ 3 files changed, 401 insertions(+) create mode 100644 .github/workflows/metadata-extract-airbyte.yml create mode 100644 .github/workflows/metadata-extract-extractors.yml create mode 100644 .github/workflows/metadata-extract-loaders.yml diff --git a/.github/workflows/metadata-extract-airbyte.yml b/.github/workflows/metadata-extract-airbyte.yml new file mode 100644 index 000000000..a155d9ddf --- /dev/null +++ b/.github/workflows/metadata-extract-airbyte.yml @@ -0,0 +1,89 @@ +name: Metadata Extract + +on: + workflow_dispatch: + inputs: + environment: + description: 'The environment to run the workflow in' + required: true + type: choice + default: 'preview' + options: + - 'preview' + - 'production' + schedule: + - cron: '0 9 * * 1' # Run at midnight UTC on Mondays + +jobs: + get_variants_list: + runs-on: ubuntu-latest + outputs: + airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }} + + steps: + - uses: actions/checkout@v4.1.7 + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + - name: Get Variants List - Airbyte + id: get-variants-list-airbyte + run: echo "VARIANTS_AIRBYTE=$(hub-utils get-variant-names $(pwd) --metadata-type=airbyte)" >> $GITHUB_OUTPUT + + - name: Set Dynamic Airbyte Matrix + id: setmatrix_airbyte + run: | + matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}" + echo "::set-output name=airbyte_matrix::$matrixStringifiedObject" + + metadata_extract_airbyte_p1: + name: Airbyte - Part 1 Metadata Extract + needs: get_variants_list + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} + container: + image: airbyte/${{ matrix.source-name }}:latest + steps: + - name: Get Airbyte Spec + run: | + $AIRBYTE_ENTRYPOINT spec | grep '^.*"type":\s*"SPEC".*$' > extractor--${{ matrix.source-name }}--airbyte.json + id: get-airbyte-stdout + - uses: actions/upload-artifact@v3 + with: + name: extractor--${{ matrix.source-name }}--airbyte.json + path: extractor--${{ matrix.source-name }}--airbyte.json + + metadata_extract_airbyte_p2: + name: Airbyte - Part 2 Metadata Extract + if: ${{ always() }} + needs: [metadata_extract_airbyte_p1, get_variants_list] + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} + env: + AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} + permissions: + id-token: write # This is required for requesting the JWT + steps: + - uses: actions/checkout@v4.1.7 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: us-west-2 + role-session-name: "GitHubActions" + + - uses: actions/download-artifact@v3 + with: + name: extractor--${{ matrix.source-name }}--airbyte.json + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + - name: Upload Metadata + run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json diff --git a/.github/workflows/metadata-extract-extractors.yml b/.github/workflows/metadata-extract-extractors.yml new file mode 100644 index 000000000..8cb00a9d3 --- /dev/null +++ b/.github/workflows/metadata-extract-extractors.yml @@ -0,0 +1,190 @@ +name: Metadata Extract + +on: + workflow_dispatch: + inputs: + environment: + description: 'The environment to run the workflow in' + required: true + type: choice + default: 'preview' + options: + - 'preview' + - 'production' + schedule: + - cron: '0 9 * * 1' # Run at midnight UTC on Mondays + +jobs: + get_variants_list: + runs-on: ubuntu-latest + outputs: + sdk_ex_matrix: ${{ steps.setmatrix_sdk_ex.outputs.sdk_ex_matrix }} + + steps: + - uses: actions/checkout@v4.1.7 + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + - name: Get Variants List - SDK Extractors (p1) + id: get-variants-list-sdk-ex-p1 + run: echo "VARIANTS_SDK_EX_P1=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk --limit=200)" >> $GITHUB_OUTPUT + + - name: Set Dynamic Matrix - SDK Extractors (p1) + id: setmatrix_sdk_ex_p1 + run: | + matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex-p1.outputs.VARIANTS_SDK_EX_P1 }}}" + echo "::set-output name=sdk_ex_p1_matrix::$matrixStringifiedObject" + + - name: Get Variants List - SDK Extractors (p2) + id: get-variants-list-sdk-ex-p2 + run: echo "VARIANTS_SDK_EX_P2=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk --skip=200)" >> $GITHUB_OUTPUT + + - name: Set Dynamic Matrix - SDK Extractors (p2) + id: setmatrix_sdk_ex_p2 + run: | + matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex-p2.outputs.VARIANTS_SDK_EX_P2 }}}" + echo "::set-output name=sdk_ex_p2_matrix::$matrixStringifiedObject" + + metadata_extract_sdk_ex_p1: + name: SDK Extractors - Metadata Extract (p1) + needs: get_variants_list + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} + env: + AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_p1_matrix) }} + permissions: + id-token: write # This is required for requesting the JWT + steps: + - uses: actions/checkout@v4.1.7 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: us-west-2 + role-session-name: "GitHubActions" + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + # This starts the attempts to install and extract metadata. Since we don't + # know what python versions are accepted we start with the latest and continue + # decreasing until we have a success or run out of python versions to attempt. + - name: Install Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Extract Metadata (3.12) + id: extract_312 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.11 (Only if 3.12 failed) + if: always() && (steps.extract_312.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Extract Metadata (3.11) + if: always() && (steps.extract_312.outcome == 'failure') + id: extract_311 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.10 (Only if 3.11 failed) + if: always() && (steps.extract_311.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Extract Metadata (3.10) + if: always() && (steps.extract_311.outcome == 'failure') + id: extract_310 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.9 (Only if 3.10 failed) + if: always() && (steps.extract_310.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Extract Metadata (3.9) + if: always() && (steps.extract_310.outcome == 'failure') + id: extract_39 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.8 (Only if 3.9 failed) + if: always() && (steps.extract_39.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.8' + + - name: Extract Metadata (3.8) + if: always() && (steps.extract_39.outcome == 'failure') + id: extract_38 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + + metadata_extract_sdk_ex_p2: + name: SDK Extractors - Metadata Extract (p2) + needs: get_variants_list + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} + env: + AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_p2_matrix) }} + permissions: + id-token: write # This is required for requesting the JWT + steps: + - uses: actions/checkout@v4.1.7 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: us-west-2 + role-session-name: "GitHubActions" + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + - name: Install Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Extract Metadata (3.10) + id: extract_310 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.9 (Only if 3.10 failed) + if: always() && (steps.extract_310.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Extract Metadata (3.9) + if: always() && (steps.extract_310.outcome == 'failure') + id: extract_39 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.8 (Only if 3.9 failed) + if: always() && (steps.extract_39.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.8' + + - name: Extract Metadata (3.8) + if: always() && (steps.extract_39.outcome == 'failure') + id: extract_38 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data diff --git a/.github/workflows/metadata-extract-loaders.yml b/.github/workflows/metadata-extract-loaders.yml new file mode 100644 index 000000000..2c9180c50 --- /dev/null +++ b/.github/workflows/metadata-extract-loaders.yml @@ -0,0 +1,122 @@ +name: Metadata Extract + +on: + workflow_dispatch: + inputs: + environment: + description: 'The environment to run the workflow in' + required: true + type: choice + default: 'preview' + options: + - 'preview' + - 'production' + schedule: + - cron: '0 9 * * 1' # Run at midnight UTC on Mondays + +jobs: + get_variants_list: + runs-on: ubuntu-latest + outputs: + sdk_loaders_matrix: ${{ steps.setmatrix_sdk_loaders.outputs.sdk_loaders_matrix }} + + steps: + - uses: actions/checkout@v4.1.7 + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + - name: Get Variants List - SDK Loaders + id: get-variants-list-sdk-loaders + run: echo "VARIANTS_SDK_LOADERS=$(hub-utils get-variant-names $(pwd) --plugin-type=loaders --metadata-type=sdk)" >> $GITHUB_OUTPUT + + - name: Set Dynamic Matrix - SDK Loaders + id: setmatrix_sdk_loaders + run: | + matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-loaders.outputs.VARIANTS_SDK_LOADERS }}}" + echo "::set-output name=sdk_loaders_matrix::$matrixStringifiedObject" + + metadata_extract_sdk_loaders: + name: SDK Loaders - Metadata Extract + needs: get_variants_list + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} + env: + AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_loaders_matrix) }} + permissions: + id-token: write # This is required for requesting the JWT + steps: + - uses: actions/checkout@v4.1.7 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: us-west-2 + role-session-name: "GitHubActions" + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + # This starts the attempts to install and extract metadata. Since we don't + # know what python versions are accepted we start with the latest and continue + # decreasing until we have a success or run out of python versions to attempt. + - name: Install Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Extract Metadata (3.12) + id: extract_312 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.11 (Only if 3.12 failed) + if: always() && (steps.extract_312.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Extract Metadata (3.11) + if: always() && (steps.extract_312.outcome == 'failure') + id: extract_311 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.10 (Only if 3.11 failed) + if: always() && (steps.extract_311.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Extract Metadata (3.10) + if: always() && (steps.extract_311.outcome == 'failure') + id: extract_310 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.9 (Only if 3.10 failed) + if: always() && (steps.extract_310.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Extract Metadata (3.9) + if: always() && (steps.extract_310.outcome == 'failure') + id: extract_39 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.8 (Only if 3.9 failed) + if: always() && (steps.extract_39.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.8' + + - name: Extract Metadata (3.8) + if: always() && (steps.extract_39.outcome == 'failure') + id: extract_38 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data From 2737280121c6d60b259816a89524552faa9078af Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Thu, 19 Sep 2024 10:56:09 -0400 Subject: [PATCH 2/3] fix actions --- .../workflows/metadata-extract-airbyte.yml | 2 +- .../workflows/metadata-extract-extractors.yml | 2 +- .../workflows/metadata-extract-loaders.yml | 2 +- .github/workflows/metadata-extract.yml | 228 ------------------ 4 files changed, 3 insertions(+), 231 deletions(-) delete mode 100644 .github/workflows/metadata-extract.yml diff --git a/.github/workflows/metadata-extract-airbyte.yml b/.github/workflows/metadata-extract-airbyte.yml index a155d9ddf..fa43115a2 100644 --- a/.github/workflows/metadata-extract-airbyte.yml +++ b/.github/workflows/metadata-extract-airbyte.yml @@ -1,4 +1,4 @@ -name: Metadata Extract +name: Metadata Extract - Airbyte on: workflow_dispatch: diff --git a/.github/workflows/metadata-extract-extractors.yml b/.github/workflows/metadata-extract-extractors.yml index 8cb00a9d3..cc9294e87 100644 --- a/.github/workflows/metadata-extract-extractors.yml +++ b/.github/workflows/metadata-extract-extractors.yml @@ -1,4 +1,4 @@ -name: Metadata Extract +name: Metadata Extract - Singer Extractors on: workflow_dispatch: diff --git a/.github/workflows/metadata-extract-loaders.yml b/.github/workflows/metadata-extract-loaders.yml index 2c9180c50..754c5742b 100644 --- a/.github/workflows/metadata-extract-loaders.yml +++ b/.github/workflows/metadata-extract-loaders.yml @@ -1,4 +1,4 @@ -name: Metadata Extract +name: Metadata Extract - Loaders on: workflow_dispatch: diff --git a/.github/workflows/metadata-extract.yml b/.github/workflows/metadata-extract.yml deleted file mode 100644 index 11987d58b..000000000 --- a/.github/workflows/metadata-extract.yml +++ /dev/null @@ -1,228 +0,0 @@ -name: Metadata Extract - -on: - workflow_dispatch: - inputs: - environment: - description: 'The environment to run the workflow in' - required: true - type: choice - default: 'preview' - options: - - 'preview' - - 'production' - schedule: - - cron: '0 9 * * 1' # Run at midnight UTC on Mondays - -jobs: - get_variants_list: - runs-on: ubuntu-latest - outputs: - sdk_ex_matrix: ${{ steps.setmatrix_sdk_ex.outputs.sdk_ex_matrix }} - sdk_loaders_matrix: ${{ steps.setmatrix_sdk_loaders.outputs.sdk_loaders_matrix }} - airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }} - - steps: - - uses: actions/checkout@v4.1.7 - - - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git - - - name: Get Variants List - SDK Extractors - id: get-variants-list-sdk-ex - run: echo "VARIANTS_SDK_EX=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk)" >> $GITHUB_OUTPUT - - - name: Set Dynamic Matrix - SDK Extractors - id: setmatrix_sdk_ex - run: | - matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex.outputs.VARIANTS_SDK_EX }}}" - echo "::set-output name=sdk_ex_matrix::$matrixStringifiedObject" - - - name: Get Variants List - SDK Loaders - id: get-variants-list-sdk-loaders - run: echo "VARIANTS_SDK_LOADERS=$(hub-utils get-variant-names $(pwd) --plugin-type=loaders --metadata-type=sdk)" >> $GITHUB_OUTPUT - - - name: Set Dynamic Matrix - SDK Loaders - id: setmatrix_sdk_loaders - run: | - matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-loaders.outputs.VARIANTS_SDK_LOADERS }}}" - echo "::set-output name=sdk_loaders_matrix::$matrixStringifiedObject" - - - name: Get Variants List - Airbyte - id: get-variants-list-airbyte - run: echo "VARIANTS_AIRBYTE=$(hub-utils get-variant-names $(pwd) --metadata-type=airbyte)" >> $GITHUB_OUTPUT - - - name: Set Dynamic Airbyte Matrix - id: setmatrix_airbyte - run: | - matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}" - echo "::set-output name=airbyte_matrix::$matrixStringifiedObject" - - metadata_extract_sdk_ex: - name: SDK Extractors - Metadata Extract - needs: get_variants_list - runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} - env: - AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_matrix) }} - permissions: - id-token: write # This is required for requesting the JWT - steps: - - uses: actions/checkout@v4.1.7 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: us-west-2 - role-session-name: "GitHubActions" - - - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git - - - name: Install Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Extract Metadata (3.10) - id: extract_310 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data - continue-on-error: true - - - name: Install Python 3.9 (Only if 3.10 failed) - if: always() && (steps.extract_310.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - name: Extract Metadata (3.9) - if: always() && (steps.extract_310.outcome == 'failure') - id: extract_39 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data - continue-on-error: true - - - name: Install Python 3.8 (Only if 3.9 failed) - if: always() && (steps.extract_39.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - - name: Extract Metadata (3.8) - if: always() && (steps.extract_39.outcome == 'failure') - id: extract_38 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data - - metadata_extract_sdk_loaders: - name: SDK Loaders - Metadata Extract - needs: get_variants_list - runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} - env: - AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_loaders_matrix) }} - permissions: - id-token: write # This is required for requesting the JWT - steps: - - uses: actions/checkout@v4.1.7 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: us-west-2 - role-session-name: "GitHubActions" - - - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git - - - name: Install Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Extract Metadata (3.10) - id: extract_310 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data - continue-on-error: true - - - name: Install Python 3.9 (Only if 3.10 failed) - if: always() && (steps.extract_310.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - name: Extract Metadata (3.9) - if: always() && (steps.extract_310.outcome == 'failure') - id: extract_39 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data - continue-on-error: true - - - name: Install Python 3.8 (Only if 3.9 failed) - if: always() && (steps.extract_39.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - - name: Extract Metadata (3.8) - if: always() && (steps.extract_39.outcome == 'failure') - id: extract_38 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data - - - metadata_extract_airbyte_p1: - name: Airbyte - Part 1 Metadata Extract - needs: get_variants_list - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} - container: - image: airbyte/${{ matrix.source-name }}:latest - steps: - - name: Get Airbyte Spec - run: | - $AIRBYTE_ENTRYPOINT spec | grep '^.*"type":\s*"SPEC".*$' > extractor--${{ matrix.source-name }}--airbyte.json - id: get-airbyte-stdout - - uses: actions/upload-artifact@v3 - with: - name: extractor--${{ matrix.source-name }}--airbyte.json - path: extractor--${{ matrix.source-name }}--airbyte.json - - metadata_extract_airbyte_p2: - name: Airbyte - Part 2 Metadata Extract - if: ${{ always() }} - needs: [metadata_extract_airbyte_p1, get_variants_list] - runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} - env: - AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} - permissions: - id-token: write # This is required for requesting the JWT - steps: - - uses: actions/checkout@v4.1.7 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: us-west-2 - role-session-name: "GitHubActions" - - - uses: actions/download-artifact@v3 - with: - name: extractor--${{ matrix.source-name }}--airbyte.json - - - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git - - - name: Upload Metadata - run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json From 526e48b0211e4497217fd2667b304cfd0959cba8 Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Thu, 19 Sep 2024 11:26:18 -0400 Subject: [PATCH 3/3] fix: Hub bot fix (#1850) --- .github/workflows/metadata-extract-extractors.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/metadata-extract-extractors.yml b/.github/workflows/metadata-extract-extractors.yml index cc9294e87..72906fe8b 100644 --- a/.github/workflows/metadata-extract-extractors.yml +++ b/.github/workflows/metadata-extract-extractors.yml @@ -18,7 +18,8 @@ jobs: get_variants_list: runs-on: ubuntu-latest outputs: - sdk_ex_matrix: ${{ steps.setmatrix_sdk_ex.outputs.sdk_ex_matrix }} + sdk_ex_p1_matrix: ${{ steps.setmatrix_sdk_ex_p1.outputs.sdk_ex_p1_matrix }} + sdk_ex_p2_matrix: ${{ steps.setmatrix_sdk_ex_p2.outputs.sdk_ex_p2_matrix }} steps: - uses: actions/checkout@v4.1.7