From fd2a0283a7a96e0901fd0e8c1ba3c8bf14e2c79d Mon Sep 17 00:00:00 2001 From: Flook Peter Date: Mon, 15 Jul 2024 13:03:30 +0800 Subject: [PATCH] Make persisting data an option, by default, don't persist data, make healthcheck have 5s interval and retry 5 times --- README.md | 8 +++- docker-compose-persist.yaml | 93 +++++++++++++++++++++++++++++++++++++ docker-compose.yaml | 62 +------------------------ run.sh | 35 ++++++++------ 4 files changed, 123 insertions(+), 75 deletions(-) create mode 100644 docker-compose-persist.yaml diff --git a/README.md b/README.md index f33b3e7..26287a2 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,12 @@ mysql mysql:3306 localhost:3306 host.docker.internal:3306 ./run.sh list ``` +### Persist data + +```shell +./run.sh -p postgres +``` + ### Remove persisted data ```shell @@ -87,7 +93,7 @@ run at startup. ### Persisted data -If any data is persisted from the services to carry across sessions, it gets pushed to folder: +If any data is persisted (via running with `-p`) from the services to carry across sessions, it gets pushed to folder: `./data//persist` diff --git a/docker-compose-persist.yaml b/docker-compose-persist.yaml new file mode 100644 index 0000000..868b7fe --- /dev/null +++ b/docker-compose-persist.yaml @@ -0,0 +1,93 @@ +"services": + "activemq": + "volumes": + - "./data/activemq/persist:/var/lib/artemis-instance" + "cassandra-server": + "volumes": + - "./data/cassandra/persist:/var/lib/cassandra" + "clickhouse-server": + "volumes": + - "./data/clickhouse/persist:/var/lib/clickhouse" + "cockroachdb": + "volumes": + - "./data/cockroachdb/persist:/cockroach/cockroach-data" + "dagster": + "volumes": + - "./data/dagster/persist:/opt/dagster/dagster_home/" + "druid": + "volumes": + - "./data/druid/persist/router_var:/opt/druid/var" + "druid-broker": + "volumes": + - "./data/druid/persist/broker_var:/opt/druid/var" + "druid-coordinator": + "volumes": + - "./data/druid/persist/shared:/opt/shared" + - "./data/druid/persist/coordinator_var:/opt/druid/var" + "druid-historical": + "volumes": + - "./data/druid/persist/shared:/opt/shared" + - "./data/druid/persist/historical_var:/opt/druid/var" + "druid-middlemanager": + "volumes": + - "./data/druid/persist/shared:/opt/shared" + - "./data/druid/persist/middle_var:/opt/druid/var" + "elasticsearch": + "volumes": + - "./data/elasticsearch/persist:/usr/share/elasticsearch/data:Z" + "flight-sql": + "volumes": + - "./data/flight-sql/persist:/opt/data" + "jupyter": + "volumes": + - "./data/jupyter/persist:/home/jovyan/work" + "kafka-server": + "volumes": + - "./data/kafka/persist/data:/var/lib/kafka/data" + - "./data/kafka/persist/etc:/etc/kafka" + "mage-ai": + "volumes": + - "./data/mage-ai/persist:/home/src/" + "mariadb": + "volumes": + - "./data/mariadb/persist:/var/lib/mysql:Z" + "marquez-server": + "volumes": + - "./data/marquez/persist:/opt/marquez" + "minio": + "volumes": + - "./data/minio/persist:/data" + "mongodb-server": + "volumes": + - "./data/mongodb/persist:/data/db" + "mssql": + "volumes": + - "./data/mssql/persist:/var/opt/mssql/data" + "mysql-server": + "volumes": + - "./data/mysql/persist:/var/lib/mysql" + "neo4j": + "volumes": + - "./data/neo4j/persist:/data" + "postgres-server": + "volumes": + - "./data/postgres/persist:/data/postgres" + "prefect-server": + "volumes": + - "./data/prefect/persist:/root/.prefect" + "rabbitmq": + "volumes": + - "./data/rabbitmq/persist:/var/lib/rabbitmq" + "solace-server": + "volumes": + - "./data/solace/persist:/var/lib/solace" + "sqlite": + "volumes": + - "./data/sqlite/persist:/opt/data" + "temporal": + "volumes": + - "./data/temporal/persist:/opt/data/db" + "unitycatalog": + "volumes": + - "./data/unitycatalog/persist:/opt/app/etc" +"version": "3.9" diff --git a/docker-compose.yaml b/docker-compose.yaml index 5cf31b5..c8a6545 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -13,8 +13,6 @@ "ports": - "61616:61616" - "8161:8161" - "volumes": - - "./data/activemq/persist:/var/lib/artemis-instance" "airflow": "command": "standalone" "container_name": "airflow" @@ -127,8 +125,6 @@ - "9042:9042" "ulimits": "memlock": -1 - "volumes": - - "./data/cassandra/persist:/var/lib/cassandra" "clickhouse": "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] "container_name": "clickhouse-data" @@ -157,8 +153,6 @@ - "8123:8123" - "9000:9000" "user": "101:101" - "volumes": - - "./data/clickhouse/persist:/var/lib/clickhouse" "cockroachdb": "command": ["start-single-node", "--insecure"] "container_name": "cockroachdb" @@ -171,8 +165,6 @@ "ports": - "26257:26257" - "8080:8080" - "volumes": - - "./data/cockroachdb/persist:/cockroach/cockroach-data" "dagster": "container_name": "dagster" "depends_on": @@ -189,7 +181,6 @@ "ports": - "3000:3000" "volumes": - - "./data/dagster/persist:/opt/dagster/dagster_home/" - "./data/dagster:/opt/dagster/app/" "data-caterer": "container_name": "data-caterer" @@ -279,8 +270,6 @@ "image": "apache/druid:${DRUID_VERSION:-30.0.0}" "ports": - "8888:8888" - "volumes": - - "./data/druid/persist/router_var:/opt/druid/var" "druid-broker": "command": ["broker"] "container_name": "druid-broker" @@ -303,8 +292,6 @@ "image": "apache/druid:${DRUID_VERSION:-30.0.0}" "ports": - "8082:8082" - "volumes": - - "./data/druid/persist/broker_var:/opt/druid/var" "druid-coordinator": "command": ["coordinator"] "container_name": "druid-coordinator" @@ -325,9 +312,6 @@ "image": "apache/druid:${DRUID_VERSION:-30.0.0}" "ports": - "8081:8081" - "volumes": - - "./data/druid/persist/shared:/opt/shared" - - "./data/druid/persist/coordinator_var:/opt/druid/var" "druid-historical": "command": ["historical"] "container_name": "druid-historical" @@ -350,9 +334,6 @@ "image": "apache/druid:${DRUID_VERSION:-30.0.0}" "ports": - "8083:8083" - "volumes": - - "./data/druid/persist/shared:/opt/shared" - - "./data/druid/persist/historical_var:/opt/druid/var" "druid-middlemanager": "command": ["middleManager"] "container_name": "druid-middlemanager" @@ -376,9 +357,6 @@ "ports": - "8091:8091" - "8100-8105:8100-8105" - "volumes": - - "./data/druid/persist/shared:/opt/shared" - - "./data/druid/persist/middle_var:/opt/druid/var" "duckdb": "container_name": "duckdb" "depends_on": @@ -402,7 +380,6 @@ "restart": "unless-stopped" "volumes": - "./data/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,Z" - - "./data/elasticsearch/persist:/usr/share/elasticsearch/data:Z" "flight-sql": "command": ["tail", "-f", "/dev/null"] "container_name": "flight-sql" @@ -416,8 +393,6 @@ "image": "voltrondata/flight-sql:${FLIGHT_SQL_VERSION:-v1.4.0}" "ports": - "31337:31337" - "volumes": - - "./data/flight-sql/persist:/opt/data" "flink": "command": "taskmanager" "container_name": "flink" @@ -455,8 +430,6 @@ "image": "quay.io/jupyter/minimal-notebook:2024-07-02" "ports": - "8888:8888" - "volumes": - - "./data/jupyter/persist:/home/jovyan/work" "kafka": "container_name": "kafka-data" "depends_on": @@ -466,7 +439,6 @@ "environment": - "KAFKA_TOPICS=${KAFKA_TOPICS:-accounts,transactions}" "image": "confluentinc/confluent-local:${KAFKA_VERSION:-7.6.1}" - "user": "${CURRENT_UID}" "volumes": - "./data/kafka/init.sh:/tmp/scripts/init.sh" "kafka-server": @@ -479,17 +451,13 @@ "expose": - "29092" "healthcheck": - "interval": "15s" - "retries": 3 + "interval": "5s" + "retries": 5 "test": ["CMD-SHELL", "/bin/sh", "-c", "kafka-topics", "--bootstrap-server", "kafka:29092", "--list"] "timeout": "5s" "image": "confluentinc/confluent-local:7.6.1" "ports": - "9092:9092" - "user": "${CURRENT_UID}" - "volumes": - - "./data/kafka/persist/data:/var/lib/kafka/data" - - "./data/kafka/persist/etc:/etc/kafka" "keycloak": "command": ["start-dev", "--import-realm"] "container_name": "keycloak" @@ -519,8 +487,6 @@ "ports": - "6789:6789" "restart": "on-failure" - "volumes": - - "./data/mage-ai/persist:/home/src/" "mariadb": "container_name": "mariadb" "environment": @@ -532,8 +498,6 @@ "ports": - "3306:3306" "restart": "always" - "volumes": - - "./data/mariadb/persist:/var/lib/mysql:Z" "marquez": "container_name": "marquez-web" "depends_on": @@ -581,7 +545,6 @@ - "5002:5000" - "5001:5001" "volumes": - - "./data/marquez/persist:/opt/marquez" - "./data/marquez/conf:/opt/app" "minio": "command": ["server", "/data", "--console-address", ":9001"] @@ -598,8 +561,6 @@ "ports": - "9000:9000" - "9001:9001" - "volumes": - - "./data/minio/persist:/data" "mongodb": "command": ["/bin/sh", "-c", "/opt/app/my_data.sh"] "container_name": "mongodb-connect" @@ -618,8 +579,6 @@ "image": "mongo:${MONGODB_VERSION:-7.0.11}" "ports": - "27017:27017" - "volumes": - - "./data/mongodb/persist:/data/db" "mssql": "container_name": "mssql" "environment": @@ -635,7 +594,6 @@ - "1433:1433" "volumes": - "./data/mssql/mssql-health-check:/usr/local/bin/mssql-health-check" - # - "./data/mssql/persist:/var/opt/mssql/data" "mysql": "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] "container_name": "mysql-data" @@ -661,8 +619,6 @@ "image": "mysql:${MYSQL_VERSION:-8.4.0}" "ports": - "3306:3306" - "volumes": - - "./data/mysql/persist:/var/lib/mysql" "neo4j": "container_name": "neo4j" "environment": @@ -678,8 +634,6 @@ "ports": - "7474:7474" - "7687:7687" - "volumes": - - "./data/neo4j/persist:/data" "pinot": "command": "StartServer -zkAddress zookeeper:2181" "container_name": "pinot-server" @@ -758,8 +712,6 @@ "image": "postgres:${POSTGRES_VERSION:-16.3}" "ports": - "5432:5432" - "volumes": - - "./data/postgres/persist:/data/postgres" "prefect": "container_name": "prefect-data" "depends_on": @@ -787,8 +739,6 @@ "ports": - "4200:4200" "restart": "always" - "volumes": - - "./data/prefect/persist:/root/.prefect" "presto": "container_name": "presto" "depends_on": @@ -815,8 +765,6 @@ "ports": - "5672:5672" - "15672:15672" - "volumes": - - "./data/rabbitmq/persist:/var/lib/rabbitmq" "solace": "container_name": "solace-data" "depends_on": @@ -851,8 +799,6 @@ "nofile": "hard": 6592 "soft": 2448 - "volumes": - - "./data/solace/persist:/var/lib/solace" "spanner": "container_name": "spanner" "image": "gcr.io/cloud-spanner-emulator/emulator:${SPANNER_VERSION:-1.5.19}" @@ -878,8 +824,6 @@ - "8233:8233" - "7233:7233" - "9233:9233" - "volumes": - - "./data/temporal/persist:/opt/data/db" "trino": "container_name": "trino" "depends_on": @@ -896,8 +840,6 @@ "image": "datacatering/unitycatalog:${UNITYCATALOG_VERSION:-0.1.0}" "ports": - "8081:8081" - "volumes": - - "./data/unitycatalog/persist:/opt/app/etc" "zookeeper": "container_name": "zookeeper" "environment": diff --git a/run.sh b/run.sh index 8680de7..1242b34 100755 --- a/run.sh +++ b/run.sh @@ -7,6 +7,7 @@ LIGHT_BLUE='\033[1;34m' NC='\033[0m' SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +COMPOSE_FILES="-f $SCRIPT_DIR/docker-compose.yaml" connection_commands=" activemq='/var/lib/artemis-instance/bin/artemis shell --user ${ARTEMIS_USER:-artemis} --password ${ARTEMIS_PASSWORD:-artemis}' @@ -39,6 +40,7 @@ usage() { echo " -d, down [services...] Shutdown services (if empty, shutdown all services)" echo " -h, --help, help Show help" echo " -l, list List supported services" + echo " -p Run service(s) with persisted data" echo " -r, remove [services...] Remove persisted data (if empty, remove all services persisted data)" echo echo "Examples:" @@ -46,18 +48,11 @@ usage() { echo " $(basename "$0") postgres Spin up Postgres" echo " $(basename "$0") -c postgres Connect to Postgres" echo " $(basename "$0") -d Bring Postgres down" + echo " $(basename "$0") -p postgres Run Postgres with persisted data" echo " $(basename "$0") -r postgres Remove Postgres persisted data" exit 0 } -set_user() { - if [ -z "$CURRENT_UID" ] - then - current_user="$(id -u):$(id -g)" - export CURRENT_UID="$current_user" - fi -} - connect_to_service() { if [ -z "$1" ] then @@ -80,7 +75,6 @@ connect_to_service() { } shutdown_service() { - set_user if [ -z "$1" ]; then echo "Shutting down all services..." docker-compose -f "$SCRIPT_DIR/docker-compose.yaml" down @@ -107,11 +101,22 @@ check_docker_installed() { fi } +check_persist_flag() { + if [ "$1" = "-p" ]; then + echo -e "${YELLOW}Persisting data to host" + COMPOSE_FILES="${COMPOSE_FILES} -f docker-compose-persist.yaml" + shift + else + echo -e "${YELLOW}Not persisting data to host" + fi + original_services=("$@") + service_array=("$@") +} + startup_services() { - set_user - all_services=("$@") echo -e "${GREEN}Starting up services...${NC}" - docker-compose -f "$SCRIPT_DIR/docker-compose.yaml" up -d --quiet-pull "$@" + echo "$COMPOSE_FILES" + docker-compose $COMPOSE_FILES up -d --quiet-pull "$@" if [ $? != 0 ]; then echo -e "${RED}Error: Failed to start up services${NC}" exit 1 @@ -122,7 +127,7 @@ startup_services() { log_how_to_connect() { echo -e "${GREEN}How to connect:${NC}" connect_result=("${YELLOW}Service,${YELLOW}Container To Container,Host To Container,Container To Host") - for service in "${all_services[@]}"; do + for service in "${original_services[@]}"; do ports=$(docker inspect "$service" | grep HostPort | sed -nr 's/.*\: "([0-9]+)"/\1/p' | sort -u) for port in $ports; do container_port=$(docker inspect "$service" | grep -B 3 "HostPort\": \"${port}\"" | sed -nr 's/.*\"([0-9]+)\/tcp\".*/\1/p' | head -1) @@ -179,7 +184,9 @@ case $1 in usage else check_docker_installed - startup_services "$@" + service_array=() + check_persist_flag "$@" + startup_services "${service_array[@]}" log_how_to_connect fi ;;