feat: use docker in frontend GHA to parallelize work (#31490)

.dockerignore

@@ -34,7 +34,6 @@
 **/*.sqllite
 **/*.swp
 **/.terser-plugin-cache/
-**/.storybook/
 **/node_modules/
 
 tests/

.github/workflows/superset-docs-verify.yml

@@ -24,7 +24,7 @@ jobs:
       - uses: JustinBeckwith/[email protected]
         continue-on-error: true # This will make the job advisory (non-blocking, no red X)
         with:
-          paths: "**/*.md, **/*.mdx"
+          paths: "**/*.md, **/*.mdx, !superset-frontend/CHANGELOG.md"
           linksToSkip: >-
             ^https://github.com/apache/(superset|incubator-superset)/(pull|issue)/\d+,
             http://localhost:8088/,

.github/workflows/superset-frontend.yml

@@ -1,4 +1,4 @@
-name: Frontend
+name: "Frontend Build CI (unit tests, linting & sanity checks)"
 
 on:
   push:
@@ -13,68 +13,166 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
   cancel-in-progress: true
 
+env:
+  TAG: apache/superset:GHA-${{ github.run_id }}
+
 jobs:
   frontend-build:
     runs-on: ubuntu-24.04
     steps:
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+      - name: Checkout Code
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          submodules: recursive
-      - name: Check npm lock file version
-        run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json
-      - name: Check for file changes
+
+      - name: Check for File Changes
         id: check
         uses: ./.github/actions/change-detector/
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Setup Node.js
-        if: steps.check.outputs.frontend
-        uses: actions/setup-node@v4
-        with:
-          node-version: "20"
-      - name: Install dependencies
-        if: steps.check.outputs.frontend
-        uses: ./.github/actions/cached-dependencies
-        with:
-          run: npm-install
-      - name: eslint
-        if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
-        run: |
-          npm run eslint -- . --quiet
-      - name: tsc
+
+      - name: Build Docker Image
         if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          npm run type
-      - name: Build plugins packages
-        if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
-        run: npm run plugins:build
-      - name: Build plugins Storybook
-        if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
-        run: npm run plugins:build-storybook
-      - name: superset-ui/core coverage
+          docker buildx build \
+            -t $TAG \
+            --cache-from=type=registry,ref=apache/superset-cache:3.10-slim-bookworm \
+            --target superset-node-ci \
+            .
+
+      - name: Save Docker Image as Artifact
         if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
         run: |
-          npm run core:cover
-      - name: unit tests
+          docker save $TAG | gzip > docker-image.tar.gz
+
+      - name: Upload Docker Image Artifact
         if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-image
+          path: docker-image.tar.gz
+
+  sharded-jest-tests:
+    needs: frontend-build
+    if: needs.frontend-build.result == 'success'
+    strategy:
+      matrix:
+        shard: [1, 2, 3, 4, 5, 6, 7, 8]
+      fail-fast: false
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download Docker Image Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker Image
+        run: docker load < docker-image.tar.gz
+
+      - name: npm run test with coverage
         run: |
-          npm run test -- --coverage --silent
-      # todo: remove this step when fix generator as a project in root jest.config.js
-      - name: generator-superset unit tests
-        if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend/packages/generator-superset
-        run: npm run test
-      - name: Upload code coverage
+          mkdir -p ${{ github.workspace }}/superset-frontend/coverage
+          docker run \
+          -v ${{ github.workspace }}/superset-frontend/coverage:/app/superset-frontend/coverage \
+          --rm $TAG \
+          bash -c \
+          "npm run test -- --coverage --shard=${{ matrix.shard }}/8 --coverageReporters=json-summary"
+
+      - name: Upload Coverage Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-artifacts-${{ matrix.shard }}
+          path: superset-frontend/coverage
+
+  report-coverage:
+    needs: [sharded-jest-tests]
+    if: needs.frontend-build.result == 'success'
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download Coverage Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-artifacts-*
+          path: coverage/
+
+      - name: Show Files
+        run: find coverage/
+
+      - name: Merge Code Coverage
+        run: npx nyc merge coverage/ merged-output/coverage-summary.json
+
+      - name: Upload Code Coverage
         uses: codecov/codecov-action@v5
         with:
           flags: javascript
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
+          files: merged-output/coverage-summary.json
+          slug: apache/superset
+
+  core-cover:
+    needs: frontend-build
+    if: needs.frontend-build.result == 'success'
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download Docker Image Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker Image
+        run: docker load < docker-image.tar.gz
+
+      - name: superset-ui/core coverage
+        run: |
+          docker run --rm $TAG bash -c \
+          "npm run core:cover"
+
+  lint-frontend:
+    needs: frontend-build
+    if: needs.frontend-build.result == 'success'
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download Docker Image Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker Image
+        run: docker load < docker-image.tar.gz
+
+      - name: eslint
+        run: |
+          docker run --rm $TAG bash -c \
+          "npm i && npm run eslint -- . --quiet"
+
+      - name: tsc
+        run: |
+          docker run --rm $TAG bash -c \
+          "npm run type"
+
+  validate-frontend:
+    needs: frontend-build
+    if: needs.frontend-build.result == 'success'
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download Docker Image Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker Image
+        run: docker load < docker-image.tar.gz
+
+      - name: Build Plugins Packages
+        run: |
+          docker run --rm $TAG bash -c \
+          "npm run plugins:build"
+
+      - name: Build Plugins Storybook
+        run: |
+          docker run --rm $TAG bash -c \
+          "npm run plugins:build-storybook"

Dockerfile

@@ -24,9 +24,9 @@ ARG PY_VER=3.10-slim-bookworm
 ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
 
 ######################################################################
-# superset-node used for building frontend assets
+# superset-node-ci used as a base for building frontend assets and CI
 ######################################################################
-FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
+FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node-ci
 ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
 ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
 ARG DEV_MODE="false"           # Skip frontend build in dev mode
@@ -53,6 +53,10 @@ RUN mkdir -p /app/superset/static/assets \
              /app/superset/translations
 
 # Mount package files and install dependencies if not in dev mode
+# NOTE: we mount packages and plugins as they are referenced in package.json as workspaces
+# ideally we'd COPY only their package.json. Here npm ci will be cached as long
+# as the full content of these folders don't change, yielding a decent cache reuse rate.
+# Note that's it's not possible selectively COPY of mount using blobs.
 RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \
     --mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \
     --mount=type=cache,target=/root/.cache \
@@ -66,6 +70,11 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j
 # Runs the webpack build process
 COPY superset-frontend /app/superset-frontend
 
+######################################################################
+# superset-node used for compile frontend assets
+######################################################################
+FROM superset-node-ci AS superset-node
+
 # Build the frontend if not in dev mode
 RUN --mount=type=cache,target=/app/superset-frontend/.temp_cache \
     --mount=type=cache,target=/root/.npm \

superset/config.py

@@ -136,7 +136,7 @@ def _try_json_readsha(filepath: str, length: int) -> str | None:
 # generated on install via setup.py. In the event that we're
 # actually running Superset, we will have already installed,
 # therefore it WILL exist. When unit tests are running, however,
-# it WILL NOT exist, so we fall back to reading package.json
+# it WILL NOT exist, so we fall back on reading package.json
 VERSION_STRING = _try_json_readversion(VERSION_INFO_FILE) or _try_json_readversion(
     PACKAGE_JSON_FILE
 )

superset/migrations/versions/2025-01-07_16-03_eb1c288c71c4_.py

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""empty message
+
+Revision ID: eb1c288c71c4
+Revises: ('df3d7e2eb9a4', '7b17aa722e30')
+Create Date: 2025-01-07 16:03:44.936921
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "eb1c288c71c4"
+down_revision = ("df3d7e2eb9a4", "7b17aa722e30")
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass