Merge branch 'main' into edgarrmondragon/feat/add-tap-fedidb

meltano · Sep 19, 2024 · 9d0af69 · 9d0af69
2 parents a2f7d47 + 177d878
commit 9d0af69
Show file tree

Hide file tree

Showing 5 changed files with 306 additions and 90 deletions.
diff --git a/.github/workflows/metadata-extract-airbyte.yml b/.github/workflows/metadata-extract-airbyte.yml
@@ -0,0 +1,89 @@
+name: Metadata Extract - Airbyte
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'The environment to run the workflow in'
+        required: true
+        type: choice
+        default: 'preview'
+        options:
+          - 'preview'
+          - 'production'
+  schedule:
+    - cron: '0 9 * * 1' # Run at midnight UTC on Mondays
+
+jobs:
+  get_variants_list:
+    runs-on: ubuntu-latest
+    outputs:
+      airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }}
+
+    steps:
+      - uses: actions/[email protected]
+
+      - name: Install hub-utils
+        run: pipx install git+https://github.com/meltano/hub-utils.git
+
+      - name: Get Variants List - Airbyte
+        id: get-variants-list-airbyte
+        run: echo "VARIANTS_AIRBYTE=$(hub-utils get-variant-names $(pwd) --metadata-type=airbyte)" >> $GITHUB_OUTPUT
+
+      - name: Set Dynamic Airbyte Matrix
+        id: setmatrix_airbyte
+        run: |
+           matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}"
+           echo "::set-output name=airbyte_matrix::$matrixStringifiedObject"
+
+  metadata_extract_airbyte_p1:
+    name: Airbyte - Part 1 Metadata Extract 
+    needs: get_variants_list
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }}
+    container:
+      image: airbyte/${{ matrix.source-name }}:latest
+    steps:
+      - name: Get Airbyte Spec
+        run: |
+          $AIRBYTE_ENTRYPOINT spec | grep '^.*"type":\s*"SPEC".*$' > extractor--${{ matrix.source-name }}--airbyte.json
+        id: get-airbyte-stdout
+      - uses: actions/upload-artifact@v3
+        with:
+          name: extractor--${{ matrix.source-name }}--airbyte.json
+          path: extractor--${{ matrix.source-name }}--airbyte.json
+
+  metadata_extract_airbyte_p2:
+    name: Airbyte - Part 2 Metadata Extract
+    if: ${{ always() }}
+    needs: [metadata_extract_airbyte_p1, get_variants_list]
+    runs-on: ubuntu-latest
+    environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }}
+    env:
+      AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}"
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }}
+    permissions:
+      id-token: write # This is required for requesting the JWT
+    steps:
+      - uses: actions/[email protected]
+
+      - name: Configure AWS credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: us-west-2
+          role-session-name: "GitHubActions"
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: extractor--${{ matrix.source-name }}--airbyte.json
+
+      - name: Install hub-utils
+        run: pipx install git+https://github.com/meltano/hub-utils.git
+
+      - name: Upload Metadata
+        run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json
diff --git a/.github/workflows/metadata-extract.yml → ...workflows/metadata-extract-extractors.yml b/.github/workflows/metadata-extract.yml → ...workflows/metadata-extract-extractors.yml
@@ -1,4 +1,4 @@
-name: Metadata Extract
+name: Metadata Extract - Singer Extractors
 
 on:
   workflow_dispatch:
@@ -18,56 +18,45 @@ jobs:
   get_variants_list:
     runs-on: ubuntu-latest
     outputs:
-      sdk_ex_matrix: ${{ steps.setmatrix_sdk_ex.outputs.sdk_ex_matrix }}
-      sdk_loaders_matrix: ${{ steps.setmatrix_sdk_loaders.outputs.sdk_loaders_matrix }}
-      airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }}
+      sdk_ex_p1_matrix: ${{ steps.setmatrix_sdk_ex_p1.outputs.sdk_ex_p1_matrix }}
+      sdk_ex_p2_matrix: ${{ steps.setmatrix_sdk_ex_p2.outputs.sdk_ex_p2_matrix }}
 
     steps:
       - uses: actions/[email protected]
 
       - name: Install hub-utils
         run: pipx install git+https://github.com/meltano/hub-utils.git
 
-      - name: Get Variants List - SDK Extractors
-        id: get-variants-list-sdk-ex
-        run: echo "VARIANTS_SDK_EX=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk)" >> $GITHUB_OUTPUT
+      - name: Get Variants List - SDK Extractors (p1)
+        id: get-variants-list-sdk-ex-p1
+        run: echo "VARIANTS_SDK_EX_P1=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk --limit=200)" >> $GITHUB_OUTPUT
 
-      - name: Set Dynamic Matrix - SDK Extractors
-        id: setmatrix_sdk_ex
+      - name: Set Dynamic Matrix - SDK Extractors (p1)
+        id: setmatrix_sdk_ex_p1
         run: |
-           matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex.outputs.VARIANTS_SDK_EX }}}"
-           echo "::set-output name=sdk_ex_matrix::$matrixStringifiedObject"
+           matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex-p1.outputs.VARIANTS_SDK_EX_P1 }}}"
+           echo "::set-output name=sdk_ex_p1_matrix::$matrixStringifiedObject"
 
-      - name: Get Variants List - SDK Loaders
-        id: get-variants-list-sdk-loaders
-        run: echo "VARIANTS_SDK_LOADERS=$(hub-utils get-variant-names $(pwd) --plugin-type=loaders --metadata-type=sdk)" >> $GITHUB_OUTPUT
+      - name: Get Variants List - SDK Extractors (p2)
+        id: get-variants-list-sdk-ex-p2
+        run: echo "VARIANTS_SDK_EX_P2=$(hub-utils get-variant-names $(pwd) --plugin-type=extractors --metadata-type=sdk --skip=200)" >> $GITHUB_OUTPUT
 
-      - name: Set Dynamic Matrix - SDK Loaders
-        id: setmatrix_sdk_loaders
+      - name: Set Dynamic Matrix - SDK Extractors (p2)
+        id: setmatrix_sdk_ex_p2
         run: |
-           matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-loaders.outputs.VARIANTS_SDK_LOADERS }}}"
-           echo "::set-output name=sdk_loaders_matrix::$matrixStringifiedObject"
+           matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex-p2.outputs.VARIANTS_SDK_EX_P2 }}}"
+           echo "::set-output name=sdk_ex_p2_matrix::$matrixStringifiedObject"
 
-      - name: Get Variants List - Airbyte
-        id: get-variants-list-airbyte
-        run: echo "VARIANTS_AIRBYTE=$(hub-utils get-variant-names $(pwd) --metadata-type=airbyte)" >> $GITHUB_OUTPUT
-
-      - name: Set Dynamic Airbyte Matrix
-        id: setmatrix_airbyte
-        run: |
-           matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}"
-           echo "::set-output name=airbyte_matrix::$matrixStringifiedObject"
-
-  metadata_extract_sdk_ex:
-    name: SDK Extractors - Metadata Extract
+  metadata_extract_sdk_ex_p1:
+    name: SDK Extractors - Metadata Extract (p1)
     needs: get_variants_list
     runs-on: ubuntu-latest
     environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }}
     env:
       AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}"
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_matrix) }}
+      matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_p1_matrix) }}
     permissions:
       id-token: write # This is required for requesting the JWT
     steps:
@@ -83,12 +72,39 @@ jobs:
     - name: Install hub-utils
       run: pipx install git+https://github.com/meltano/hub-utils.git
 
-    - name: Install Python 3.10
+    # This starts the attempts to install and extract metadata. Since we don't
+    # know what python versions are accepted we start with the latest and continue
+    # decreasing until we have a success or run out of python versions to attempt.
+    - name: Install Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Extract Metadata (3.12)
+      id: extract_312
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      continue-on-error: true
+
+    - name: Install Python 3.11 (Only if 3.12 failed)
+      if: always() && (steps.extract_312.outcome == 'failure')
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
+    - name: Extract Metadata (3.11)
+      if: always() && (steps.extract_312.outcome == 'failure')
+      id: extract_311
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      continue-on-error: true
+
+    - name: Install Python 3.10 (Only if 3.11 failed)
+      if: always() && (steps.extract_311.outcome == 'failure')
       uses: actions/setup-python@v5
       with:
         python-version: '3.10'
 
     - name: Extract Metadata (3.10)
+      if: always() && (steps.extract_311.outcome == 'failure')
       id: extract_310
       run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
       continue-on-error: true
@@ -116,16 +132,16 @@ jobs:
       id: extract_38
       run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
 
-  metadata_extract_sdk_loaders:
-    name: SDK Loaders - Metadata Extract
+  metadata_extract_sdk_ex_p2:
+    name: SDK Extractors - Metadata Extract (p2)
     needs: get_variants_list
     runs-on: ubuntu-latest
     environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }}
     env:
       AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}"
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_loaders_matrix) }}
+      matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_ex_p2_matrix) }}
     permissions:
       id-token: write # This is required for requesting the JWT
     steps:
@@ -173,56 +189,3 @@ jobs:
       if: always() && (steps.extract_39.outcome == 'failure')
       id: extract_38
       run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
-
-
-  metadata_extract_airbyte_p1:
-    name: Airbyte - Part 1 Metadata Extract 
-    needs: get_variants_list
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }}
-    container:
-      image: airbyte/${{ matrix.source-name }}:latest
-    steps:
-      - name: Get Airbyte Spec
-        run: |
-          $AIRBYTE_ENTRYPOINT spec | grep '^.*"type":\s*"SPEC".*$' > extractor--${{ matrix.source-name }}--airbyte.json
-        id: get-airbyte-stdout
-      - uses: actions/upload-artifact@v3
-        with:
-          name: extractor--${{ matrix.source-name }}--airbyte.json
-          path: extractor--${{ matrix.source-name }}--airbyte.json
-
-  metadata_extract_airbyte_p2:
-    name: Airbyte - Part 2 Metadata Extract
-    if: ${{ always() }}
-    needs: [metadata_extract_airbyte_p1, get_variants_list]
-    runs-on: ubuntu-latest
-    environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }}
-    env:
-      AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}"
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.get_variants_list.outputs.airbyte_matrix) }}
-    permissions:
-      id-token: write # This is required for requesting the JWT
-    steps:
-      - uses: actions/[email protected]
-
-      - name: Configure AWS credentials
-        uses: aws-actions/[email protected]
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: us-west-2
-          role-session-name: "GitHubActions"
-
-      - uses: actions/download-artifact@v3
-        with:
-          name: extractor--${{ matrix.source-name }}--airbyte.json
-
-      - name: Install hub-utils
-        run: pipx install git+https://github.com/meltano/hub-utils.git
-
-      - name: Upload Metadata
-        run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json