ci: Use the correct Python executables for extracting plugin metadata (…

…#1883) Co-authored-by: MeltyBot <[email protected]> Co-authored-by: edgarrmondragon <[email protected]>
meltano · Nov 5, 2024 · e6ce9fe · e6ce9fe
1 parent a7ef09a
commit e6ce9fe
Show file tree

Hide file tree

Showing 8 changed files with 97 additions and 93 deletions.
diff --git a/.github/workflows/metadata-extract-airbyte.yml b/.github/workflows/metadata-extract-airbyte.yml
@@ -24,7 +24,7 @@ jobs:
       - uses: actions/[email protected]
 
       - name: Install hub-utils
-        run: pipx install git+https://github.com/meltano/hub-utils.git
+        run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable
 
       - name: Get Variants List - Airbyte
         id: get-variants-list-airbyte
@@ -34,7 +34,7 @@ jobs:
         id: setmatrix_airbyte
         run: |
            matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}"
-           echo "::set-output name=airbyte_matrix::$matrixStringifiedObject"
+           echo "airbyte_matrix=$matrixStringifiedObject" >> $GITHUB_ENV
 
   metadata_extract_airbyte_p1:
     name: Airbyte - Part 1 Metadata Extract 
@@ -83,7 +83,7 @@ jobs:
           name: extractor--${{ matrix.source-name }}--airbyte.json
 
       - name: Install hub-utils
-        run: pipx install git+https://github.com/meltano/hub-utils.git
+        run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable
 
       - name: Upload Metadata
         run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json
diff --git a/.github/workflows/metadata-extract-extractors.yml b/.github/workflows/metadata-extract-extractors.yml
@@ -25,7 +25,7 @@ jobs:
       - uses: actions/[email protected]
 
       - name: Install hub-utils
-        run: pipx install git+https://github.com/meltano/hub-utils.git
+        run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable
 
       - name: Get Variants List - SDK Extractors (p1)
         id: get-variants-list-sdk-ex-p1
@@ -35,7 +35,7 @@ jobs:
         id: setmatrix_sdk_ex_p1
         run: |
            matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex-p1.outputs.VARIANTS_SDK_EX_P1 }}}"
-           echo "::set-output name=sdk_ex_p1_matrix::$matrixStringifiedObject"
+           echo "sdk_ex_p1_matrix=$matrixStringifiedObject" >> $GITHUB_ENV
 
       - name: Get Variants List - SDK Extractors (p2)
         id: get-variants-list-sdk-ex-p2
@@ -45,7 +45,7 @@ jobs:
         id: setmatrix_sdk_ex_p2
         run: |
            matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex-p2.outputs.VARIANTS_SDK_EX_P2 }}}"
-           echo "::set-output name=sdk_ex_p2_matrix::$matrixStringifiedObject"
+           echo "sdk_ex_p2_matrix=$matrixStringifiedObject" >> $GITHUB_ENV
 
   metadata_extract_sdk_ex_p1:
     name: SDK Extractors - Metadata Extract (p1)
@@ -70,67 +70,55 @@ jobs:
         role-session-name: "GitHubActions"
 
     - name: Install hub-utils
-      run: pipx install git+https://github.com/meltano/hub-utils.git
+      run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable
 
     # This starts the attempts to install and extract metadata. Since we don't
     # know what python versions are accepted we start with the latest and continue
     # decreasing until we have a success or run out of python versions to attempt.
-    - name: Install Python 3.12
+    - name: Install Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
-
+        python-version: |
+          3.13
+          3.12
+          3.11
+          3.10
+          3.9
+          3.8
+
+    - name: Extract Metadata (3.13)
+      id: extract_313
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.13
+      continue-on-error: true
+
     - name: Extract Metadata (3.12)
+      if: always() && (steps.extract_313.outcome == 'failure')
       id: extract_312
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.12
       continue-on-error: true
 
-    - name: Install Python 3.11 (Only if 3.12 failed)
-      if: always() && (steps.extract_312.outcome == 'failure')
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.11'
-
     - name: Extract Metadata (3.11)
       if: always() && (steps.extract_312.outcome == 'failure')
       id: extract_311
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.11
       continue-on-error: true
 
-    - name: Install Python 3.10 (Only if 3.11 failed)
-      if: always() && (steps.extract_311.outcome == 'failure')
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.10'
-
     - name: Extract Metadata (3.10)
       if: always() && (steps.extract_311.outcome == 'failure')
       id: extract_310
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.10
       continue-on-error: true
 
-    - name: Install Python 3.9 (Only if 3.10 failed)
-      if: always() && (steps.extract_310.outcome == 'failure')
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.9'
-
     - name: Extract Metadata (3.9)
       if: always() && (steps.extract_310.outcome == 'failure')
       id: extract_39
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.9
       continue-on-error: true
 
-    - name: Install Python 3.8 (Only if 3.9 failed)
-      if: always() && (steps.extract_39.outcome == 'failure')
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.8'
-
     - name: Extract Metadata (3.8)
       if: always() && (steps.extract_39.outcome == 'failure')
       id: extract_38
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.8
 
   metadata_extract_sdk_ex_p2:
     name: SDK Extractors - Metadata Extract (p2)
@@ -155,37 +143,28 @@ jobs:
         role-session-name: "GitHubActions"
 
     - name: Install hub-utils
-      run: pipx install git+https://github.com/meltano/hub-utils.git
+      run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable
 
-    - name: Install Python 3.10
+    - name: Install Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: |
+          3.10
+          3.9
+          3.8
 
     - name: Extract Metadata (3.10)
       id: extract_310
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.10
       continue-on-error: true
 
-    - name: Install Python 3.9 (Only if 3.10 failed)
-      if: always() && (steps.extract_310.outcome == 'failure')
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.9'
-
     - name: Extract Metadata (3.9)
       if: always() && (steps.extract_310.outcome == 'failure')
       id: extract_39
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.9
       continue-on-error: true
 
-    - name: Install Python 3.8 (Only if 3.9 failed)
-      if: always() && (steps.extract_39.outcome == 'failure')
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.8'
-
     - name: Extract Metadata (3.8)
       if: always() && (steps.extract_39.outcome == 'failure')
       id: extract_38
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.8
diff --git a/.github/workflows/metadata-extract-loaders.yml b/.github/workflows/metadata-extract-loaders.yml
@@ -24,7 +24,7 @@ jobs:
       - uses: actions/[email protected]
 
       - name: Install hub-utils
-        run: pipx install git+https://github.com/meltano/hub-utils.git
+        run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable
 
       - name: Get Variants List - SDK Loaders
         id: get-variants-list-sdk-loaders
@@ -34,7 +34,7 @@ jobs:
         id: setmatrix_sdk_loaders
         run: |
            matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-loaders.outputs.VARIANTS_SDK_LOADERS }}}"
-           echo "::set-output name=sdk_loaders_matrix::$matrixStringifiedObject"
+           echo "sdk_loaders_matrix=$matrixStringifiedObject" >> $GITHUB_ENV
 
   metadata_extract_sdk_loaders:
     name: SDK Loaders - Metadata Extract
@@ -59,64 +59,52 @@ jobs:
         role-session-name: "GitHubActions"
 
     - name: Install hub-utils
-      run: pipx install git+https://github.com/meltano/hub-utils.git
+      run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable
 
     # This starts the attempts to install and extract metadata. Since we don't
     # know what python versions are accepted we start with the latest and continue
     # decreasing until we have a success or run out of python versions to attempt.
     - name: Install Python 3.12
       uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: |
+          3.13
+          3.12
+          3.11
+          3.10
+          3.9
+          3.8
+
+    - name: Extract Metadata (3.13)
+      id: extract_313
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.13
+      continue-on-error: true
 
     - name: Extract Metadata (3.12)
+      if: always() && (steps.extract_313.outcome == 'failure')
       id: extract_312
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.12
       continue-on-error: true
 
-    - name: Install Python 3.11 (Only if 3.12 failed)
-      if: always() && (steps.extract_312.outcome == 'failure')
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.11'
-
     - name: Extract Metadata (3.11)
       if: always() && (steps.extract_312.outcome == 'failure')
       id: extract_311
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.11
       continue-on-error: true
 
-    - name: Install Python 3.10 (Only if 3.11 failed)
-      if: always() && (steps.extract_311.outcome == 'failure')
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.10'
-
     - name: Extract Metadata (3.10)
       if: always() && (steps.extract_311.outcome == 'failure')
       id: extract_310
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.10
       continue-on-error: true
 
-    - name: Install Python 3.9 (Only if 3.10 failed)
-      if: always() && (steps.extract_310.outcome == 'failure')
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.9'
-
     - name: Extract Metadata (3.9)
       if: always() && (steps.extract_310.outcome == 'failure')
       id: extract_39
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.9
       continue-on-error: true
 
-    - name: Install Python 3.8 (Only if 3.9 failed)
-      if: always() && (steps.extract_39.outcome == 'failure')
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.8'
-
     - name: Extract Metadata (3.8)
       if: always() && (steps.extract_39.outcome == 'failure')
       id: extract_38
-      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data
+      run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.8
diff --git a/_data/meltano/extractors/tap-aircall/ticketswap.yml b/_data/meltano/extractors/tap-aircall/ticketswap.yml
@@ -57,6 +57,14 @@ settings:
   kind: string
   label: Batch Config Storage Root
   name: batch_config.storage.root
+- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization'
+  kind: array
+  label: Faker Config Locale
+  name: faker_config.locale
+- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator'
+  kind: string
+  label: Faker Config Seed
+  name: faker_config.seed
 - description: "'True' to enable schema flattening and automatically expand nested
     properties."
   kind: boolean

diff --git a/_data/meltano/extractors/tap-apple-search-ads/ticketswap.yml b/_data/meltano/extractors/tap-apple-search-ads/ticketswap.yml
@@ -81,6 +81,20 @@ settings:
   label: Org ID
   name: org_id
   sensitive: true
+- description: The granularity of reporting streams. One of HOURLY, DAILY, WEEKLY,
+    MONTHLY.
+  kind: options
+  label: Report Granularity
+  name: report_granularity
+  options:
+  - label: Hourly
+    value: HOURLY
+  - label: Daily
+    value: DAILY
+  - label: Weekly
+    value: WEEKLY
+  - label: Monthly
+    value: MONTHLY
 - description: Start date for reporting streams, format in YYYY-MM-DD.
   kind: date_iso8601
   label: Start Date

diff --git a/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml b/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml
@@ -92,7 +92,7 @@ settings:
   kind: string
   label: User Agent
   name: user_agent
-  value: tap-circle-ci/0.3.2 Singer tap for the CircleCI API.
+  value: tap-circle-ci/0.3.3 Singer tap for the CircleCI API.
 settings_group_validation:
 - - org_slug
   - token

diff --git a/_data/meltano/extractors/tap-dbt/meltanolabs.yml b/_data/meltano/extractors/tap-dbt/meltanolabs.yml
@@ -95,7 +95,7 @@ settings:
   kind: string
   label: User Agent
   name: user_agent
-  value: tap-dbt/0.12.3.post1.dev8+a4e0b7b Singer tap for the dbt Cloud API.
+  value: tap-dbt/0.12.3.post1 Singer tap for the dbt Cloud API.
 settings_group_validation:
 - - account_ids
   - api_key

diff --git a/_data/meltano/extractors/tap-mysql/meltanolabs.yml b/_data/meltano/extractors/tap-mysql/meltanolabs.yml
@@ -80,6 +80,12 @@ settings:
   kind: string
   label: Host
   name: host
+- description: By default we'll check if the database is a Vitess instance. If you
+    would rather not automatically check, set this to `False`. See Vitess/PlanetScale
+    documentation below for more information.
+  kind: boolean
+  label: Is Vitess
+  name: is_vitess
 - description: Password used to authenticate. Note if sqlalchemy_url is set this will
     be ignored.
   kind: password
@@ -92,7 +98,16 @@ settings:
   label: Port
   name: port
   value: 3306
-- description: Example mysql://[username]:[password]@localhost:3306/[db_name]
+- description: Sqlalchemy_url options (also called the query), to connect to PlanetScale
+    you must turn on SSL see PlanetScale information below. Note if sqlalchemy_url
+    is set this will be ignored.
+  kind: object
+  label: Sqlalchemy Options
+  name: sqlalchemy_options
+- description: Example pymysql://[username]:[password]@localhost:3306/[db_name][?options]
+    see 
+    https://docs.sqlalchemy.org/en/20/dialects/mysql.html#module-sqlalchemy.dialects.mysql.pymysql
+    for more information
   kind: string
   label: Sqlalchemy URL
   name: sqlalchemy_url