diff --git a/.dockerignore b/.dockerignore index 31c873f0073f9..b650f22c10806 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,7 +34,6 @@ **/*.sqllite **/*.swp **/.terser-plugin-cache/ -**/.storybook/ **/node_modules/ tests/ diff --git a/.github/workflows/superset-docs-verify.yml b/.github/workflows/superset-docs-verify.yml index 7fcc7309a50bb..f3d04a33c06fc 100644 --- a/.github/workflows/superset-docs-verify.yml +++ b/.github/workflows/superset-docs-verify.yml @@ -24,7 +24,7 @@ jobs: - uses: JustinBeckwith/linkinator-action@v1.11.0 continue-on-error: true # This will make the job advisory (non-blocking, no red X) with: - paths: "**/*.md, **/*.mdx" + paths: "**/*.md, **/*.mdx, !superset-frontend/CHANGELOG.md" linksToSkip: >- ^https://github.com/apache/(superset|incubator-superset)/(pull|issue)/\d+, http://localhost:8088/, diff --git a/.github/workflows/superset-frontend.yml b/.github/workflows/superset-frontend.yml index 9451692f5f7a5..4a67958663adb 100644 --- a/.github/workflows/superset-frontend.yml +++ b/.github/workflows/superset-frontend.yml @@ -1,4 +1,4 @@ -name: Frontend +name: "Frontend Build CI (unit tests, linting & sanity checks)" on: push: @@ -13,68 +13,166 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true +env: + TAG: apache/superset:GHA-${{ github.run_id }} + jobs: frontend-build: runs-on: ubuntu-24.04 steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + - name: Checkout Code uses: actions/checkout@v4 with: persist-credentials: false - submodules: recursive - - name: Check npm lock file version - run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json - - name: Check for file changes + + - name: Check for File Changes id: check uses: ./.github/actions/change-detector/ with: token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Node.js - if: steps.check.outputs.frontend - uses: actions/setup-node@v4 - with: - node-version: "20" - - name: Install dependencies - if: steps.check.outputs.frontend - uses: ./.github/actions/cached-dependencies - with: - run: npm-install - - name: eslint - if: steps.check.outputs.frontend - working-directory: ./superset-frontend - run: | - npm run eslint -- . --quiet - - name: tsc + + - name: Build Docker Image if: steps.check.outputs.frontend - working-directory: ./superset-frontend + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - npm run type - - name: Build plugins packages - if: steps.check.outputs.frontend - working-directory: ./superset-frontend - run: npm run plugins:build - - name: Build plugins Storybook - if: steps.check.outputs.frontend - working-directory: ./superset-frontend - run: npm run plugins:build-storybook - - name: superset-ui/core coverage + docker buildx build \ + -t $TAG \ + --cache-from=type=registry,ref=apache/superset-cache:3.10-slim-bookworm \ + --target superset-node-ci \ + . + + - name: Save Docker Image as Artifact if: steps.check.outputs.frontend - working-directory: ./superset-frontend run: | - npm run core:cover - - name: unit tests + docker save $TAG | gzip > docker-image.tar.gz + + - name: Upload Docker Image Artifact if: steps.check.outputs.frontend - working-directory: ./superset-frontend + uses: actions/upload-artifact@v4 + with: + name: docker-image + path: docker-image.tar.gz + + sharded-jest-tests: + needs: frontend-build + if: needs.frontend-build.result == 'success' + strategy: + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] + fail-fast: false + runs-on: ubuntu-24.04 + steps: + - name: Download Docker Image Artifact + uses: actions/download-artifact@v4 + with: + name: docker-image + + - name: Load Docker Image + run: docker load < docker-image.tar.gz + + - name: npm run test with coverage run: | - npm run test -- --coverage --silent - # todo: remove this step when fix generator as a project in root jest.config.js - - name: generator-superset unit tests - if: steps.check.outputs.frontend - working-directory: ./superset-frontend/packages/generator-superset - run: npm run test - - name: Upload code coverage + mkdir -p ${{ github.workspace }}/superset-frontend/coverage + docker run \ + -v ${{ github.workspace }}/superset-frontend/coverage:/app/superset-frontend/coverage \ + --rm $TAG \ + bash -c \ + "npm run test -- --coverage --shard=${{ matrix.shard }}/8 --coverageReporters=json-summary" + + - name: Upload Coverage Artifact + uses: actions/upload-artifact@v4 + with: + name: coverage-artifacts-${{ matrix.shard }} + path: superset-frontend/coverage + + report-coverage: + needs: [sharded-jest-tests] + if: needs.frontend-build.result == 'success' + runs-on: ubuntu-24.04 + steps: + - name: Download Coverage Artifacts + uses: actions/download-artifact@v4 + with: + pattern: coverage-artifacts-* + path: coverage/ + + - name: Show Files + run: find coverage/ + + - name: Merge Code Coverage + run: npx nyc merge coverage/ merged-output/coverage-summary.json + + - name: Upload Code Coverage uses: codecov/codecov-action@v5 with: flags: javascript token: ${{ secrets.CODECOV_TOKEN }} verbose: true + files: merged-output/coverage-summary.json + slug: apache/superset + + core-cover: + needs: frontend-build + if: needs.frontend-build.result == 'success' + runs-on: ubuntu-24.04 + steps: + - name: Download Docker Image Artifact + uses: actions/download-artifact@v4 + with: + name: docker-image + + - name: Load Docker Image + run: docker load < docker-image.tar.gz + + - name: superset-ui/core coverage + run: | + docker run --rm $TAG bash -c \ + "npm run core:cover" + + lint-frontend: + needs: frontend-build + if: needs.frontend-build.result == 'success' + runs-on: ubuntu-24.04 + steps: + - name: Download Docker Image Artifact + uses: actions/download-artifact@v4 + with: + name: docker-image + + - name: Load Docker Image + run: docker load < docker-image.tar.gz + + - name: eslint + run: | + docker run --rm $TAG bash -c \ + "npm i && npm run eslint -- . --quiet" + + - name: tsc + run: | + docker run --rm $TAG bash -c \ + "npm run type" + + validate-frontend: + needs: frontend-build + if: needs.frontend-build.result == 'success' + runs-on: ubuntu-24.04 + steps: + - name: Download Docker Image Artifact + uses: actions/download-artifact@v4 + with: + name: docker-image + + - name: Load Docker Image + run: docker load < docker-image.tar.gz + + - name: Build Plugins Packages + run: | + docker run --rm $TAG bash -c \ + "npm run plugins:build" + + - name: Build Plugins Storybook + run: | + docker run --rm $TAG bash -c \ + "npm run plugins:build-storybook" diff --git a/Dockerfile b/Dockerfile index 4f24360988101..7297ad139337b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,9 +24,9 @@ ARG PY_VER=3.10-slim-bookworm ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64} ###################################################################### -# superset-node used for building frontend assets +# superset-node-ci used as a base for building frontend assets and CI ###################################################################### -FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node +FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node-ci ARG BUILD_TRANSLATIONS="false" # Include translations in the final build ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS} ARG DEV_MODE="false" # Skip frontend build in dev mode @@ -53,6 +53,10 @@ RUN mkdir -p /app/superset/static/assets \ /app/superset/translations # Mount package files and install dependencies if not in dev mode +# NOTE: we mount packages and plugins as they are referenced in package.json as workspaces +# ideally we'd COPY only their package.json. Here npm ci will be cached as long +# as the full content of these folders don't change, yielding a decent cache reuse rate. +# Note that's it's not possible selectively COPY of mount using blobs. RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \ --mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \ --mount=type=cache,target=/root/.cache \ @@ -66,6 +70,11 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j # Runs the webpack build process COPY superset-frontend /app/superset-frontend +###################################################################### +# superset-node used for compile frontend assets +###################################################################### +FROM superset-node-ci AS superset-node + # Build the frontend if not in dev mode RUN --mount=type=cache,target=/app/superset-frontend/.temp_cache \ --mount=type=cache,target=/root/.npm \ diff --git a/superset-frontend/package-lock.json b/superset-frontend/package-lock.json index b7000a02687f5..693c8ac646658 100644 --- a/superset-frontend/package-lock.json +++ b/superset-frontend/package-lock.json @@ -57127,7 +57127,7 @@ "license": "Apache-2.0", "dependencies": { "@data-ui/event-flow": "^0.0.84", - "@emotion/cache": "^11.4.0", + "@emotion/cache": "^11.14.0", "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.0", "@mihkeleidast/storybook-addon-source": "^1.0.1", @@ -67792,7 +67792,7 @@ "@babel/preset-react": "^7.26.3", "@babel/preset-typescript": "^7.23.3", "@data-ui/event-flow": "^0.0.84", - "@emotion/cache": "^11.4.0", + "@emotion/cache": "^11.14.0", "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.0", "@mihkeleidast/storybook-addon-source": "^1.0.1", diff --git a/superset/config.py b/superset/config.py index 3fc94cd27c18e..5d03016298aae 100644 --- a/superset/config.py +++ b/superset/config.py @@ -136,7 +136,7 @@ def _try_json_readsha(filepath: str, length: int) -> str | None: # generated on install via setup.py. In the event that we're # actually running Superset, we will have already installed, # therefore it WILL exist. When unit tests are running, however, -# it WILL NOT exist, so we fall back to reading package.json +# it WILL NOT exist, so we fall back on reading package.json VERSION_STRING = _try_json_readversion(VERSION_INFO_FILE) or _try_json_readversion( PACKAGE_JSON_FILE ) diff --git a/superset/migrations/versions/2025-01-07_16-03_eb1c288c71c4_.py b/superset/migrations/versions/2025-01-07_16-03_eb1c288c71c4_.py new file mode 100644 index 0000000000000..86780f7fde5fb --- /dev/null +++ b/superset/migrations/versions/2025-01-07_16-03_eb1c288c71c4_.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""empty message + +Revision ID: eb1c288c71c4 +Revises: ('df3d7e2eb9a4', '7b17aa722e30') +Create Date: 2025-01-07 16:03:44.936921 + +""" + +# revision identifiers, used by Alembic. +revision = "eb1c288c71c4" +down_revision = ("df3d7e2eb9a4", "7b17aa722e30") + + +def upgrade(): + pass + + +def downgrade(): + pass