diff --git a/.github/workflows/metadata-extract-airbyte.yml b/.github/workflows/metadata-extract-airbyte.yml new file mode 100644 index 000000000..fa43115a2 --- /dev/null +++ b/.github/workflows/metadata-extract-airbyte.yml @@ -0,0 +1,89 @@ +name: Metadata Extract - Airbyte + +on: + workflow_dispatch: + inputs: + environment: + description: 'The environment to run the workflow in' + required: true + type: choice + default: 'preview' + options: + - 'preview' + - 'production' + schedule: + - cron: '0 9 * * 1' # Run at midnight UTC on Mondays + +jobs: + get_variants_list: + runs-on: ubuntu-latest + outputs: + airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }} + + steps: + - uses: actions/checkout@v4.1.7 + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + - name: Get Variants List - Airbyte + id: get-variants-list-airbyte + run: echo "VARIANTS_AIRBYTE=$(hub-utils get-variant-names $(pwd) --metadata-type=airbyte)" >> $GITHUB_OUTPUT + + - name: Set Dynamic Airbyte Matrix + id: setmatrix_airbyte + run: | + matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}" + echo "::set-output name=airbyte_matrix::$matrixStringifiedObject" + + metadata_extract_airbyte_p1: + name: Airbyte - Part 1 Metadata Extract + needs: get_variants_list + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} + container: + image: airbyte/${{ matrix.source-name }}:latest + steps: + - name: Get Airbyte Spec + run: | + $AIRBYTE_ENTRYPOINT spec | grep '^.*"type":\s*"SPEC".*$' > extractor--${{ matrix.source-name }}--airbyte.json + id: get-airbyte-stdout + - uses: actions/upload-artifact@v3 + with: + name: extractor--${{ matrix.source-name }}--airbyte.json + path: extractor--${{ matrix.source-name }}--airbyte.json + + metadata_extract_airbyte_p2: + name: Airbyte - Part 2 Metadata Extract + if: ${{ always() }} + needs: [metadata_extract_airbyte_p1, get_variants_list] + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} + env: + AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} + permissions: + id-token: write # This is required for requesting the JWT + steps: + - uses: actions/checkout@v4.1.7 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: us-west-2 + role-session-name: "GitHubActions" + + - uses: actions/download-artifact@v3 + with: + name: extractor--${{ matrix.source-name }}--airbyte.json + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + - name: Upload Metadata + run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json diff --git a/.github/workflows/metadata-extract.yml b/.github/workflows/metadata-extract-extractors.yml similarity index 55% rename from .github/workflows/metadata-extract.yml rename to .github/workflows/metadata-extract-extractors.yml index 11987d58b..72906fe8b 100644 --- a/.github/workflows/metadata-extract.yml +++ b/.github/workflows/metadata-extract-extractors.yml @@ -1,4 +1,4 @@ -name: Metadata Extract +name: Metadata Extract - Singer Extractors on: workflow_dispatch: @@ -18,9 +18,8 @@ jobs: get_variants_list: runs-on: ubuntu-latest outputs: - sdk_ex_matrix: ${{ steps.setmatrix_sdk_ex.outputs.sdk_ex_matrix }} - sdk_loaders_matrix: ${{ steps.setmatrix_sdk_loaders.outputs.sdk_loaders_matrix }} - airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }} + sdk_ex_p1_matrix: ${{ steps.setmatrix_sdk_ex_p1.outputs.sdk_ex_p1_matrix }} + sdk_ex_p2_matrix: ${{ steps.setmatrix_sdk_ex_p2.outputs.sdk_ex_p2_matrix }} steps: - uses: actions/checkout@v4.1.7 @@ -28,38 +27,28 @@ jobs: - name: Install hub-utils run: pipx install git+https://github.com/meltano/hub-utils.git - - name: Get Variants List - SDK Extractors - id: get-variants-list-sdk-ex - run: echo "VARIANTS_SDK_EX=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk)" >> $GITHUB_OUTPUT + - name: Get Variants List - SDK Extractors (p1) + id: get-variants-list-sdk-ex-p1 + run: echo "VARIANTS_SDK_EX_P1=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk --limit=200)" >> $GITHUB_OUTPUT - - name: Set Dynamic Matrix - SDK Extractors - id: setmatrix_sdk_ex + - name: Set Dynamic Matrix - SDK Extractors (p1) + id: setmatrix_sdk_ex_p1 run: | - matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex.outputs.VARIANTS_SDK_EX }}}" - echo "::set-output name=sdk_ex_matrix::$matrixStringifiedObject" + matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex-p1.outputs.VARIANTS_SDK_EX_P1 }}}" + echo "::set-output name=sdk_ex_p1_matrix::$matrixStringifiedObject" - - name: Get Variants List - SDK Loaders - id: get-variants-list-sdk-loaders - run: echo "VARIANTS_SDK_LOADERS=$(hub-utils get-variant-names $(pwd) --plugin-type=loaders --metadata-type=sdk)" >> $GITHUB_OUTPUT + - name: Get Variants List - SDK Extractors (p2) + id: get-variants-list-sdk-ex-p2 + run: echo "VARIANTS_SDK_EX_P2=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk --skip=200)" >> $GITHUB_OUTPUT - - name: Set Dynamic Matrix - SDK Loaders - id: setmatrix_sdk_loaders + - name: Set Dynamic Matrix - SDK Extractors (p2) + id: setmatrix_sdk_ex_p2 run: | - matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-loaders.outputs.VARIANTS_SDK_LOADERS }}}" - echo "::set-output name=sdk_loaders_matrix::$matrixStringifiedObject" + matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex-p2.outputs.VARIANTS_SDK_EX_P2 }}}" + echo "::set-output name=sdk_ex_p2_matrix::$matrixStringifiedObject" - - name: Get Variants List - Airbyte - id: get-variants-list-airbyte - run: echo "VARIANTS_AIRBYTE=$(hub-utils get-variant-names $(pwd) --metadata-type=airbyte)" >> $GITHUB_OUTPUT - - - name: Set Dynamic Airbyte Matrix - id: setmatrix_airbyte - run: | - matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}" - echo "::set-output name=airbyte_matrix::$matrixStringifiedObject" - - metadata_extract_sdk_ex: - name: SDK Extractors - Metadata Extract + metadata_extract_sdk_ex_p1: + name: SDK Extractors - Metadata Extract (p1) needs: get_variants_list runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} @@ -67,7 +56,7 @@ jobs: AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" strategy: fail-fast: false - matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_matrix) }} + matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_p1_matrix) }} permissions: id-token: write # This is required for requesting the JWT steps: @@ -83,12 +72,39 @@ jobs: - name: Install hub-utils run: pipx install git+https://github.com/meltano/hub-utils.git - - name: Install Python 3.10 + # This starts the attempts to install and extract metadata. Since we don't + # know what python versions are accepted we start with the latest and continue + # decreasing until we have a success or run out of python versions to attempt. + - name: Install Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Extract Metadata (3.12) + id: extract_312 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.11 (Only if 3.12 failed) + if: always() && (steps.extract_312.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Extract Metadata (3.11) + if: always() && (steps.extract_312.outcome == 'failure') + id: extract_311 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.10 (Only if 3.11 failed) + if: always() && (steps.extract_311.outcome == 'failure') uses: actions/setup-python@v5 with: python-version: '3.10' - name: Extract Metadata (3.10) + if: always() && (steps.extract_311.outcome == 'failure') id: extract_310 run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data continue-on-error: true @@ -116,8 +132,8 @@ jobs: id: extract_38 run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data - metadata_extract_sdk_loaders: - name: SDK Loaders - Metadata Extract + metadata_extract_sdk_ex_p2: + name: SDK Extractors - Metadata Extract (p2) needs: get_variants_list runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} @@ -125,7 +141,7 @@ jobs: AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" strategy: fail-fast: false - matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_loaders_matrix) }} + matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_p2_matrix) }} permissions: id-token: write # This is required for requesting the JWT steps: @@ -173,56 +189,3 @@ jobs: if: always() && (steps.extract_39.outcome == 'failure') id: extract_38 run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data - - - metadata_extract_airbyte_p1: - name: Airbyte - Part 1 Metadata Extract - needs: get_variants_list - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} - container: - image: airbyte/${{ matrix.source-name }}:latest - steps: - - name: Get Airbyte Spec - run: | - $AIRBYTE_ENTRYPOINT spec | grep '^.*"type":\s*"SPEC".*$' > extractor--${{ matrix.source-name }}--airbyte.json - id: get-airbyte-stdout - - uses: actions/upload-artifact@v3 - with: - name: extractor--${{ matrix.source-name }}--airbyte.json - path: extractor--${{ matrix.source-name }}--airbyte.json - - metadata_extract_airbyte_p2: - name: Airbyte - Part 2 Metadata Extract - if: ${{ always() }} - needs: [metadata_extract_airbyte_p1, get_variants_list] - runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} - env: - AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }} - permissions: - id-token: write # This is required for requesting the JWT - steps: - - uses: actions/checkout@v4.1.7 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: us-west-2 - role-session-name: "GitHubActions" - - - uses: actions/download-artifact@v3 - with: - name: extractor--${{ matrix.source-name }}--airbyte.json - - - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git - - - name: Upload Metadata - run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json diff --git a/.github/workflows/metadata-extract-loaders.yml b/.github/workflows/metadata-extract-loaders.yml new file mode 100644 index 000000000..754c5742b --- /dev/null +++ b/.github/workflows/metadata-extract-loaders.yml @@ -0,0 +1,122 @@ +name: Metadata Extract - Loaders + +on: + workflow_dispatch: + inputs: + environment: + description: 'The environment to run the workflow in' + required: true + type: choice + default: 'preview' + options: + - 'preview' + - 'production' + schedule: + - cron: '0 9 * * 1' # Run at midnight UTC on Mondays + +jobs: + get_variants_list: + runs-on: ubuntu-latest + outputs: + sdk_loaders_matrix: ${{ steps.setmatrix_sdk_loaders.outputs.sdk_loaders_matrix }} + + steps: + - uses: actions/checkout@v4.1.7 + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + - name: Get Variants List - SDK Loaders + id: get-variants-list-sdk-loaders + run: echo "VARIANTS_SDK_LOADERS=$(hub-utils get-variant-names $(pwd) --plugin-type=loaders --metadata-type=sdk)" >> $GITHUB_OUTPUT + + - name: Set Dynamic Matrix - SDK Loaders + id: setmatrix_sdk_loaders + run: | + matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-loaders.outputs.VARIANTS_SDK_LOADERS }}}" + echo "::set-output name=sdk_loaders_matrix::$matrixStringifiedObject" + + metadata_extract_sdk_loaders: + name: SDK Loaders - Metadata Extract + needs: get_variants_list + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} + env: + AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_loaders_matrix) }} + permissions: + id-token: write # This is required for requesting the JWT + steps: + - uses: actions/checkout@v4.1.7 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4.0.2 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: us-west-2 + role-session-name: "GitHubActions" + + - name: Install hub-utils + run: pipx install git+https://github.com/meltano/hub-utils.git + + # This starts the attempts to install and extract metadata. Since we don't + # know what python versions are accepted we start with the latest and continue + # decreasing until we have a success or run out of python versions to attempt. + - name: Install Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Extract Metadata (3.12) + id: extract_312 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.11 (Only if 3.12 failed) + if: always() && (steps.extract_312.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Extract Metadata (3.11) + if: always() && (steps.extract_312.outcome == 'failure') + id: extract_311 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.10 (Only if 3.11 failed) + if: always() && (steps.extract_311.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Extract Metadata (3.10) + if: always() && (steps.extract_311.outcome == 'failure') + id: extract_310 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.9 (Only if 3.10 failed) + if: always() && (steps.extract_310.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Extract Metadata (3.9) + if: always() && (steps.extract_310.outcome == 'failure') + id: extract_39 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + continue-on-error: true + + - name: Install Python 3.8 (Only if 3.9 failed) + if: always() && (steps.extract_39.outcome == 'failure') + uses: actions/setup-python@v5 + with: + python-version: '3.8' + + - name: Extract Metadata (3.8) + if: always() && (steps.extract_39.outcome == 'failure') + id: extract_38 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data diff --git a/_data/maintainers.yml b/_data/maintainers.yml index e21f3ab02..2cc580997 100644 --- a/_data/maintainers.yml +++ b/_data/maintainers.yml @@ -1168,9 +1168,9 @@ thoughtchad: name: thoughtchad url: https://github.com/thoughtchad ticketswap: - label: Ticketswap + label: TicketSwap name: ticketswap - url: https://github.com/Ticketswap + url: https://github.com/TicketSwap timeplus: label: Timeplus name: timeplus diff --git a/_data/meltano/extractors/tap-apple-search-ads/ticketswap.yml b/_data/meltano/extractors/tap-apple-search-ads/ticketswap.yml new file mode 100644 index 000000000..618074529 --- /dev/null +++ b/_data/meltano/extractors/tap-apple-search-ads/ticketswap.yml @@ -0,0 +1,42 @@ +capabilities: +- catalog +- discover +- state +domain_url: https://searchads.apple.com/ +keywords: +- meltano_sdk +label: Apple Search Ads +logo_url: /assets/logos/extractors/apple-search-ads.png +maintenance_status: active +name: tap-apple-search-ads +namespace: tap_apple_search_ads +next_steps: '' +pip_url: git+https://github.com/TicketSwap/tap-apple-search-ads.git +quality: silver +repo: https://github.com/TicketSwap/tap-apple-search-ads +settings: +- description: obtained from "Implementing OAuth for the Apple Search Ads API". + kind: password + label: Client Id + name: client_id + sensitive: true +- description: obtained from "Implementing OAuth for the Apple Search Ads API". + kind: password + label: Client Secret + name: client_secret + sensitive: true +- description: Apple Search Ads Organization ID, obtained from Apple Search Ads console. + label: Org Id + name: org_id +- description: Start date for reporting streams, format in YYYY-MM-DD. + label: Start Date + name: start_date +- description: End date for reporting streams, format in YYYY-MM-DD. + label: End Date + name: end_date +settings_group_validation: +- - client_id + client_secret +settings_preamble: '' +usage: '' +variant: ticketswap