Add in superset and redis

README.md

@@ -125,6 +125,7 @@ POSTGRES_USER=my-user POSTGRES_PASSWORD=my-password ./run.sh postgres
 | Service Type                | Service                   | Supported  |
 |-----------------------------|---------------------------|------------|
 | Api Gateway                 | kong                      | ✅         |
+| Cache                       | redis                     | ✅         |
 | Change Data Capture         | debezium                  | ✅         |
 | Code Analysis               | sonarqube                 | ✅         |
 | Database                    | cassandra                 | ✅         |
@@ -147,6 +148,7 @@ POSTGRES_USER=my-user POSTGRES_PASSWORD=my-password ./run.sh postgres
 | Data Catalog                | unitycatalog              | ✅         |
 | Data Collector              | fluentd                   | ✅         |
 | Data Collector              | logstash                  | ✅         |
+| Data Visualisation          | superset                  | ✅         |
 | Distributed Coordination    | zookeeper                 | ✅         |
 | Distributed Data Processing | flink                     | ✅         |
 | Identity Management         | keycloak                  | ✅         |

data/postgres/data/my_data.sql

@@ -52,3 +52,5 @@ CREATE DATABASE keycloak;
 CREATE DATABASE marquez;
 CREATE DATABASE druid;
 CREATE DATABASE kong;
+CREATE DATABASE superset;
+CREATE DATABASE superset_examples;

data/superset/docker/.env

@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+COMPOSE_PROJECT_NAME=superset
+
+# database configurations (do not modify)
+DATABASE_DB=superset
+DATABASE_HOST=postgres
+# Make sure you set this to a unique secure random value on production
+DATABASE_PASSWORD=postgres
+DATABASE_USER=postgres
+
+EXAMPLES_DB=superset_examples
+EXAMPLES_HOST=postgres
+EXAMPLES_USER=postgres
+# Make sure you set this to a unique secure random value on production
+EXAMPLES_PASSWORD=postgres
+EXAMPLES_PORT=5432
+
+# database engine specific environment variables
+# change the below if you prefer another database engine
+DATABASE_PORT=5432
+DATABASE_DIALECT=postgresql
+POSTGRES_DB=superset
+POSTGRES_USER=postgres
+# Make sure you set this to a unique secure random value on production
+POSTGRES_PASSWORD=postgres
+#MYSQL_DATABASE=superset
+#MYSQL_USER=superset
+#MYSQL_PASSWORD=superset
+#MYSQL_RANDOM_ROOT_PASSWORD=yes
+
+# Add the mapped in /app/pythonpath_docker which allows devs to override stuff
+PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev
+REDIS_HOST=redis
+REDIS_PORT=6379
+
+FLASK_DEBUG=true
+SUPERSET_ENV=development
+SUPERSET_LOAD_EXAMPLES=yes
+CYPRESS_CONFIG=false
+SUPERSET_PORT=8088
+MAPBOX_API_KEY=''
+
+# Make sure you set this to a unique secure random value on production
+SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET
+
+ENABLE_PLAYWRIGHT=false
+PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
+BUILD_SUPERSET_FRONTEND_IN_DOCKER=false

data/superset/docker/docker-bootstrap.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -eo pipefail
+
+REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
+#
+# Make sure we have dev requirements installed
+#
+if [ -f "${REQUIREMENTS_LOCAL}" ]; then
+  echo "Installing local overrides at ${REQUIREMENTS_LOCAL}"
+  pip install --no-cache-dir -r "${REQUIREMENTS_LOCAL}"
+else
+  echo "Skipping local overrides"
+#  pip install --no-cache-dir /app
+fi
+
+case "${1}" in
+  worker)
+    echo "Starting Celery worker..."
+    # setting up only 2 workers by default to contain memory usage in dev environments
+    celery --app=superset.tasks.celery_app:app worker -O fair -l INFO --concurrency=${CELERYD_CONCURRENCY:-2}
+    ;;
+  beat)
+    echo "Starting Celery beat..."
+    rm -f /tmp/celerybeat.pid
+    celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
+    ;;
+  app)
+    echo "Starting web app (using development server)..."
+    flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+    ;;
+  app-gunicorn)
+    echo "Starting web app..."
+    /usr/bin/run-server.sh
+    ;;
+  *)
+    echo "Unknown Operation!!!"
+    ;;
+esac

data/superset/docker/docker-init.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+#
+# Always install local overrides first
+#
+/app/docker/docker-bootstrap.sh
+
+STEP_CNT=4
+
+echo_step() {
+cat <<EOF
+
+######################################################################
+
+
+Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
+
+
+######################################################################
+
+EOF
+}
+ADMIN_PASSWORD="${ADMIN_PASSWORD:-admin}"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    ADMIN_PASSWORD="general"
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://postgres:postgres@postgres:5432/superset
+fi
+# Initialize the database
+echo_step "1" "Starting" "Applying DB migrations"
+superset db upgrade
+echo_step "1" "Complete" "Applying DB migrations"
+
+# Create an admin user
+echo_step "2" "Starting" "Setting up admin user ( admin / $ADMIN_PASSWORD )"
+superset fab create-admin \
+              --username admin \
+              --firstname Superset \
+              --lastname Admin \
+              --email [email protected] \
+              --password "$ADMIN_PASSWORD"
+echo_step "2" "Complete" "Setting up admin user"
+# Create default roles and permissions
+echo_step "3" "Starting" "Setting up roles and perms"
+superset init
+echo_step "3" "Complete" "Setting up roles and perms"
+
+if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
+    # Load some data to play with
+    echo_step "4" "Starting" "Loading examples"
+    # If Cypress run which consumes superset_test_config – load required data for tests
+    if [ "$CYPRESS_CONFIG" == "true" ]; then
+        superset load_test_users
+        superset load_examples --load-test-data
+    else
+        superset load_examples --force
+    fi
+    echo_step "4" "Complete" "Loading examples"
+fi
+
+exit 0

data/superset/docker/pythonpath_dev/superset_config.py

@@ -0,0 +1,119 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This file is included in the final Docker image and SHOULD be overridden when
+# deploying the image to prod. Settings configured here are intended for use in local
+# development environments. Also note that superset_config_docker.py is imported
+# as a final step as a means to override "defaults" configured here
+#
+import logging
+import os
+
+from celery.schedules import crontab
+from flask_caching.backends.filesystemcache import FileSystemCache
+
+logger = logging.getLogger()
+
+DATABASE_DIALECT = os.getenv("DATABASE_DIALECT")
+DATABASE_USER = os.getenv("DATABASE_USER")
+DATABASE_PASSWORD = os.getenv("DATABASE_PASSWORD")
+DATABASE_HOST = os.getenv("DATABASE_HOST")
+DATABASE_PORT = os.getenv("DATABASE_PORT")
+DATABASE_DB = os.getenv("DATABASE_DB")
+
+EXAMPLES_USER = os.getenv("EXAMPLES_USER")
+EXAMPLES_PASSWORD = os.getenv("EXAMPLES_PASSWORD")
+EXAMPLES_HOST = os.getenv("EXAMPLES_HOST")
+EXAMPLES_PORT = os.getenv("EXAMPLES_PORT")
+EXAMPLES_DB = os.getenv("EXAMPLES_DB")
+
+# The SQLAlchemy connection string.
+SQLALCHEMY_DATABASE_URI = (
+    f"{DATABASE_DIALECT}://"
+    f"{DATABASE_USER}:{DATABASE_PASSWORD}@"
+    f"{DATABASE_HOST}:{DATABASE_PORT}/{DATABASE_DB}"
+)
+
+SQLALCHEMY_EXAMPLES_URI = (
+    f"{DATABASE_DIALECT}://"
+    f"{EXAMPLES_USER}:{EXAMPLES_PASSWORD}@"
+    f"{EXAMPLES_HOST}:{EXAMPLES_PORT}/{EXAMPLES_DB}"
+)
+
+REDIS_HOST = os.getenv("REDIS_HOST", "redis")
+REDIS_PORT = os.getenv("REDIS_PORT", "6379")
+REDIS_CELERY_DB = os.getenv("REDIS_CELERY_DB", "0")
+REDIS_RESULTS_DB = os.getenv("REDIS_RESULTS_DB", "1")
+
+RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab")
+
+CACHE_CONFIG = {
+    "CACHE_TYPE": "RedisCache",
+    "CACHE_DEFAULT_TIMEOUT": 300,
+    "CACHE_KEY_PREFIX": "superset_",
+    "CACHE_REDIS_HOST": REDIS_HOST,
+    "CACHE_REDIS_PORT": REDIS_PORT,
+    "CACHE_REDIS_DB": REDIS_RESULTS_DB,
+}
+DATA_CACHE_CONFIG = CACHE_CONFIG
+
+
+class CeleryConfig:
+    broker_url = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
+    imports = (
+        "superset.sql_lab",
+        "superset.tasks.scheduler",
+        "superset.tasks.thumbnails",
+        "superset.tasks.cache",
+    )
+    result_backend = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"
+    worker_prefetch_multiplier = 1
+    task_acks_late = False
+    beat_schedule = {
+        "reports.scheduler": {
+            "task": "reports.scheduler",
+            "schedule": crontab(minute="*", hour="*"),
+        },
+        "reports.prune_log": {
+            "task": "reports.prune_log",
+            "schedule": crontab(minute=10, hour=0),
+        },
+    }
+
+
+CELERY_CONFIG = CeleryConfig
+
+FEATURE_FLAGS = {"ALERT_REPORTS": True}
+ALERT_REPORTS_NOTIFICATION_DRY_RUN = True
+WEBDRIVER_BASEURL = "http://superset:8088/"  # When using docker compose baseurl should be http://superset_app:8088/
+# The base URL for the email report hyperlinks.
+WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL
+SQLLAB_CTAS_NO_LIMIT = True
+
+#
+# Optionally import superset_config_docker.py (which will have been included on
+# the PYTHONPATH) in order to allow for local settings to be overridden
+#
+try:
+    import superset_config_docker
+    from superset_config_docker import *  # noqa
+
+    logger.info(
+        f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]"
+    )
+except ImportError:
+    logger.info("Using default Docker config...")

data/superset/docker/pythonpath_dev/superset_config_docker.py

@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# This is an example "local" configuration file. In order to set/override config
+# options that ONLY apply to your local environment, simply copy/rename this file
+# to docker/pythonpath_dev/superset_config_docker.py
+# It ends up being imported by docker/superset_config.py which is loaded by
+# superset/config.py
+#
+
+SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://postgres:postgres@postgres/superset"
+SQLALCHEMY_ECHO = True