Merge branch 'master' of https://github.com/apache/superset into apac…

…he-master

.gitattributes

@@ -1 +1,2 @@
 docker/**/*.sh text eol=lf
+*.svg binary

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -15,14 +15,9 @@ body:
     id: bug-description
     attributes:
       label: Bug description
-      description: A clear and concise description of what the bug is.
-    validations:
-      required: true
-  - type: textarea
-    id: repro-steps
-    attributes:
-      label: How to reproduce the bug
+      description: A clear description of what the bug is, including reproduction steps and expected behavior.
       placeholder: |
+        The bug is that...
         1. Go to '...'
         2. Click on '....'
         3. Scroll down to '....'

.github/workflows/bashlib.sh

@@ -162,7 +162,11 @@ cypress-run-all() {
     USE_DASHBOARD_FLAG='--use-dashboard'
   fi
 
-  python ../../scripts/cypress_run.py --parallelism $PARALLELISM --parallelism-id $PARALLEL_ID $USE_DASHBOARD_FLAG
+  # UNCOMMENT the next few commands to monitor memory usage
+  # monitor_memory &  # Start memory monitoring in the background
+  # memoryMonitorPid=$!
+  python ../../scripts/cypress_run.py --parallelism $PARALLELISM --parallelism-id $PARALLEL_ID --group $PARALLEL_ID --retries 5 $USE_DASHBOARD_FLAG
+  # kill $memoryMonitorPid
 
   # After job is done, print out Flask log for debugging
   echo "::group::Flask log for default run"
@@ -178,6 +182,21 @@ eyes-storybook-dependencies() {
   say "::endgroup::"
 }
 
+monitor_memory() {
+  # This is a small utility to monitor memory usage. Useful for debugging memory in GHA.
+  # To use wrap your command as follows
+  #
+  # monitor_memory &  # Start memory monitoring in the background
+  # memoryMonitorPid=$!
+  # YOUR_COMMAND_HERE
+  # kill $memoryMonitorPid
+  while true; do
+    echo "$(date) - Top 5 memory-consuming processes:"
+    ps -eo pid,comm,%mem --sort=-%mem | head -n 6  # First line is the header, next 5 are top processes
+    sleep 2
+  done
+}
+
 cypress-run-applitools() {
   cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
 

.github/workflows/ephemeral-env.yml

@@ -233,7 +233,7 @@ jobs:
 
     - name: Deploy Amazon ECS task definition
       id: deploy-task
-      uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+      uses: aws-actions/amazon-ecs-deploy-task-definition@v2
       with:
         task-definition: ${{ steps.task-def.outputs.task-definition }}
         service: pr-${{ github.event.issue.number }}-service

.github/workflows/superset-docs-verify.yml

@@ -16,6 +16,7 @@ jobs:
     # See docs here: https://github.com/marketplace/actions/linkinator
     name: Link Checking
     runs-on: ubuntu-latest
+    continue-on-error: true # This will make the job advisory (non-blocking, no red X)
     steps:
       - uses: actions/checkout@v4
       - uses: JustinBeckwith/[email protected]

.github/workflows/superset-e2e.yml

@@ -48,7 +48,8 @@ jobs:
       PYTHONPATH: ${{ github.workspace }}
       REDIS_PORT: 16379
       GITHUB_TOKEN: ${{ github.token }}
-      USE_DASHBOARD: ${{ github.event.inputs.use_dashboard || (github.ref == 'refs/heads/master' && 'true') || 'false' }}
+      # use the dashboard feature when running manually OR merging to master
+      USE_DASHBOARD: ${{ github.event.inputs.use_dashboard == 'true'|| (github.ref == 'refs/heads/master' && 'true') || 'false' }}
     services:
       postgres:
         image: postgres:15-alpine
@@ -131,6 +132,7 @@ jobs:
           PARALLEL_ID: ${{ matrix.parallel_id }}
           PARALLELISM: 6
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          NODE_OPTIONS: "--max-old-space-size=4096"
         with:
           run: cypress-run-all ${{ env.USE_DASHBOARD }}
       - name: Upload Artifacts

.github/workflows/superset-frontend.yml

@@ -49,11 +49,6 @@ jobs:
         working-directory: ./superset-frontend
         run: |
           npm run type
-      - name: prettier
-        if: steps.check.outputs.frontend
-        working-directory: ./superset-frontend
-        run: |
-          npm run prettier-check
       - name: Build plugins packages
         if: steps.check.outputs.frontend
         working-directory: ./superset-frontend

.pre-commit-config.yaml

@@ -53,6 +53,7 @@ repos:
       - id: debug-statements
       - id: end-of-file-fixer
       - id: trailing-whitespace
+        exclude: ^.*\.(snap)
         args: ["--markdown-linebreak-ext=md"]
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.1.0 # Use the sha or tag you want to point at

.rat-excludes

@@ -61,6 +61,9 @@ tsconfig.tsbuildinfo
 generator-superset/*
 temporary_superset_ui/*
 
+# skip license checks for auto-generated test snapshots
+.*snap
+
 # docs overrides for third party logos we don't have the rights to
 google-big-query.svg
 google-sheets.svg

CODE_OF_CONDUCT.md

@@ -80,9 +80,9 @@ If you believe someone is violating this code of conduct, you may reply to them
 
 Or one of our volunteers:
 
-* [Mark Thomas](http://home.apache.org/~markt/coc.html)
-* [Joan Touzet](http://home.apache.org/~wohali/)
-* [Sharan Foga](http://home.apache.org/~sharan/coc.html)
+* [Mark Thomas](https://www.linkedin.com/in/mark-thomas-b16751158/)
+* [Joan Touzet](https://www.apache.org/foundation/conduct-team/wohali.html)
+* [Sharan Foga](https://www.linkedin.com/in/sfoga/)
 
 If the violation is in documentation or code, for example inappropriate pronoun usage or word choice within official documentation, we ask that people report these privately to the project in question at <[email protected]>, and, if they have sufficient ability within the project, to resolve or remove the concerning material, being mindful of the perspective of the person originally reporting the issue.
 

Dockerfile

@@ -26,12 +26,25 @@ FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
 
 ARG NPM_BUILD_CMD="build"
 
+# Include translations in the final build. The default supports en only to
+# reduce complexity and weight for those only using en
+ARG BUILD_TRANSLATIONS="false"
+
+# Used by docker-compose to skip the frontend build,
+# in dev we mount the repo and build the frontend inside docker
+ARG DEV_MODE="false"
+
+# Include headless browsers? Allows for alerts, reports & thumbnails, but bloats the images
+ARG INCLUDE_CHROMIUM="true"
+ARG INCLUDE_FIREFOX="false"
+
 # Somehow we need python3 + build-essential on this side of the house to install node-gyp
 RUN apt-get update -qq \
     && apt-get install \
         -yqq --no-install-recommends \
         build-essential \
-        python3
+        python3 \
+        zstd
 
 ENV BUILD_CMD=${NPM_BUILD_CMD} \
     PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
@@ -41,18 +54,34 @@ RUN --mount=type=bind,target=/frontend-mem-nag.sh,src=./docker/frontend-mem-nag.
     /frontend-mem-nag.sh
 
 WORKDIR /app/superset-frontend
+# Creating empty folders to avoid errors when running COPY later on
+RUN mkdir -p /app/superset/static/assets
 RUN --mount=type=bind,target=./package.json,src=./superset-frontend/package.json \
     --mount=type=bind,target=./package-lock.json,src=./superset-frontend/package-lock.json \
-    npm ci
+    if [ "$DEV_MODE" = "false" ]; then \
+        npm ci; \
+    else \
+        echo "Skipping 'npm ci' in dev mode"; \
+    fi
 
 # Runs the webpack build process
 COPY superset-frontend /app/superset-frontend
-
 # This copies the .po files needed for translation
 RUN mkdir -p /app/superset/translations
 COPY superset/translations /app/superset/translations
+RUN if [ "$DEV_MODE" = "false" ]; then \
+        BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \
+    else \
+        echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \
+    fi
+
+
 # Compiles .json files from the .po files, then deletes the .po files
-RUN npm run build-translation
+RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
+        npm run build-translation; \
+    else \
+        echo "Skipping translations as requested by build flag"; \
+    fi
 RUN rm /app/superset/translations/*/LC_MESSAGES/*.po
 RUN rm /app/superset/translations/messages.pot
 
@@ -63,6 +92,10 @@ RUN npm run ${BUILD_CMD}
 ######################################################################
 FROM python:${PY_VER} AS lean
 
+# Include translations in the final build. The default supports en only to
+# reduce complexity and weight for those only using en
+ARG BUILD_TRANSLATIONS="false"
+
 WORKDIR /app
 ENV LANG=C.UTF-8 \
     LC_ALL=C.UTF-8 \
@@ -76,7 +109,6 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache
     && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
     && apt-get update -qq && apt-get install -yqq --no-install-recommends \
         curl \
-        default-libmysqlclient-dev \
         libsasl2-dev \
         libsasl2-modules-gssapi-mit \
         libpq-dev \
@@ -93,8 +125,8 @@ COPY --chown=superset:superset requirements/base.txt requirements/
 RUN --mount=type=cache,target=/root/.cache/pip \
     apt-get update -qq && apt-get install -yqq --no-install-recommends \
       build-essential \
-    && pip install --upgrade setuptools pip \
-    && pip install -r requirements/base.txt \
+    && pip install --no-cache-dir --upgrade setuptools pip \
+    && pip install --no-cache-dir -r requirements/base.txt \
     && apt-get autoremove -yqq --purge build-essential \
     && rm -rf /var/lib/apt/lists/*
 
@@ -104,17 +136,21 @@ COPY --chown=superset:superset --from=superset-node /app/superset/static/assets
 ## Lastly, let's install superset itself
 COPY --chown=superset:superset superset superset
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install -e .
+    pip install --no-cache-dir -e .
 
 # Copy the .json translations from the frontend layer
 COPY --chown=superset:superset --from=superset-node /app/superset/translations superset/translations
 
 # Compile translations for the backend - this generates .mo files, then deletes the .po files
 COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/
-RUN ./scripts/translations/generate_mo_files.sh \
-    && chown -R superset:superset superset/translations \
-    && rm superset/translations/messages.pot \
-    && rm superset/translations/*/LC_MESSAGES/*.po
+RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
+        ./scripts/translations/generate_mo_files.sh \
+        && chown -R superset:superset superset/translations \
+        && rm superset/translations/messages.pot \
+        && rm superset/translations/*/LC_MESSAGES/*.po; \
+    else \
+        echo "Skipping translations as requested by build flag"; \
+    fi
 
 COPY --chmod=755 ./docker/run-server.sh /usr/bin/
 USER superset
@@ -144,28 +180,38 @@ RUN apt-get update -qq \
         && rm -rf /var/lib/apt/lists/*
 
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install playwright
+    pip install --no-cache-dir playwright
 RUN playwright install-deps
-RUN playwright install chromium
+
+RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \
+        playwright install chromium; \
+    else \
+        echo "Skipping translations in dev mode"; \
+    fi
 
 # Install GeckoDriver WebDriver
 ARG GECKODRIVER_VERSION=v0.34.0 \
     FIREFOX_VERSION=125.0.3
 
-RUN apt-get update -qq \
-    && apt-get install -yqq --no-install-recommends wget bzip2 \
-    && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
-    # Install Firefox
-    && wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
-    && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
-    && apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/*
-# Cache everything for dev purposes...
+RUN if [ "$INCLUDE_FIREFOX" = "true" ]; then \
+        apt-get update -qq \
+        && apt-get install -yqq --no-install-recommends wget bzip2 \
+        && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
+        && wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
+        && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
+        && apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/*; \
+    fi
+
+# Installing mysql client os-level dependencies in dev image only because GPL
+RUN apt-get install -yqq --no-install-recommends \
+        default-libmysqlclient-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --chown=superset:superset requirements/development.txt requirements/
 RUN --mount=type=cache,target=/root/.cache/pip \
     apt-get update -qq && apt-get install -yqq --no-install-recommends \
       build-essential \
-    && pip install -r requirements/development.txt \
+    && pip install --no-cache-dir -r requirements/development.txt \
     && apt-get autoremove -yqq --purge build-essential \
     && rm -rf /var/lib/apt/lists/*
 

README.md

@@ -134,6 +134,7 @@ Here are some of the major database solutions that are supported:
   <img src="https://superset.apache.org/img/databases/starrocks.png" alt="starrocks" border="0" width="200" />
   <img src="https://superset.apache.org/img/databases/doris.png" alt="doris" border="0" width="200" />
   <img src="https://superset.apache.org/img/databases/oceanbase.svg" alt="oceanbase" border="0" width="220" />
+  <img src="https://superset.apache.org/img/databases/sap-hana.png" alt="oceanbase" border="0" width="220" />
 </p>
 
 **A more comprehensive list of supported databases** along with the configuration instructions can be found [here](https://superset.apache.org/docs/configuration/databases).

RELEASING/release-notes-4-1/README.md

@@ -19,7 +19,11 @@ under the License.
 
 # Release Notes for Superset 4.1.0
 
-Superset 4.1.0 brings a range of new features and quality of life improvements.  This release is a minor version, meaning it doesn't include any breaking changes to ensure a seamless transition for our users. Here are some of the highlights of this release.
+Superset 4.1.0 brings a range of new features and quality of life improvements.  This release is a minor version, meaning it doesn't include any breaking changes. However, users of basic Superset docker images like `4.1.0` should see the note at the bottom of this file about [changes to those builds](/RELEASING/release-notes-4-1/README.md#change-to-docker-image-builds).
+
+## Highlights
+
+Here are some of the highlights of this release.
 
 ### Big Number With Time Period Updates
 
@@ -129,3 +133,8 @@ Users can now see both the [total and percentage in tooltips](https://github.com
 ### Additional Metadata Bar To Dashboards
 
 There is now a [metadata bar](https://github.com/apache/superset/pull/27857) added to the header of dashboards. This will now show viewers of the dashboard both the owners and last modified time of the dashboard.
+
+
+## Change to Docker image builds
+
+Starting in 4.1.0, the release's docker image does not ship with drivers needed to operate Superset. Users may need to install a driver for their metadata database (MySQL or Postgres) as well as the driver for their data warehouse. This is a result of changes to the `lean` docker image that official releases come from; see [Docker Build Presets](/docs/installation/docker-builds#build-presets) for more details.

RESOURCES/FEATURE_FLAGS.md

@@ -68,6 +68,13 @@ These features flags are **safe for production**. They have been tested and will
 - DISABLE_LEGACY_DATASOURCE_EDITOR
 
 ### Flags retained for runtime configuration
+
+Currently some of our feature flags act as dynamic configurations that can changed
+on the fly. This acts in contradiction with the typical ephemeral feature flag use case,
+where the flag is used to mature a feature, and eventually deprecated once the feature is
+solid. Eventually we'll likely refactor these under a more formal "dynamic configurations" managed
+independently. This new framework will also allow for non-boolean configurations.
+
 - ALERTS_ATTACH_REPORTS
 - ALLOW_ADHOC_SUBQUERY
 - DASHBOARD_RBAC [(docs)](https://superset.apache.org/docs/using-superset/creating-your-first-dashboard#manage-access-to-dashboards)
@@ -79,6 +86,7 @@ These features flags are **safe for production**. They have been tested and will
 - ESCAPE_MARKDOWN_HTML
 - LISTVIEWS_DEFAULT_CARD_VIEW
 - SCHEDULED_QUERIES [(docs)](https://superset.apache.org/docs/configuration/alerts-reports)
+- SLACK_ENABLE_AVATARS (see `superset/config.py` for more information)
 - SQLLAB_BACKEND_PERSISTENCE
 - SQL_VALIDATORS_BY_ENGINE [(docs)](https://superset.apache.org/docs/configuration/sql-templating)
 - THUMBNAILS [(docs)](https://superset.apache.org/docs/configuration/cache)