From 2d2431602a0eea22a2559c040c2d1e9b0331ef0f Mon Sep 17 00:00:00 2001 From: Flook Peter Date: Thu, 20 Jun 2024 12:54:51 +0800 Subject: [PATCH 1/2] Add in remove persisted data command, allow for user and password to be overridden via env variables, define consistent way of defining custom data --- README.md | 31 ++- data/cassandra/{ => cql}/my_data.cql | 0 data/cassandra/init.sh | 16 ++ .../1_create_postgres_connection.sh | 10 - data/clickhouse/init.sh | 15 ++ .../sql/create_postgres_connection.sql | 5 + data/druid/environment | 2 - data/kafka/init.sh | 18 ++ data/kafka/my_data.sh | 11 -- data/mysql/init.sh | 16 ++ data/mysql/{ => sql}/my_data.sql | 2 +- data/postgres/init.sh | 16 ++ data/postgres/{ => sql}/my_data.sql | 0 docker-compose.yaml | 184 ++++++++++++------ run.sh | 59 ++++-- 15 files changed, 286 insertions(+), 99 deletions(-) rename data/cassandra/{ => cql}/my_data.cql (100%) create mode 100755 data/cassandra/init.sh delete mode 100755 data/clickhouse/docker-entrypoint-initdb.d/1_create_postgres_connection.sh create mode 100755 data/clickhouse/init.sh create mode 100644 data/clickhouse/sql/create_postgres_connection.sql create mode 100755 data/kafka/init.sh delete mode 100755 data/kafka/my_data.sh create mode 100755 data/mysql/init.sh rename data/mysql/{ => sql}/my_data.sql (97%) create mode 100755 data/postgres/init.sh rename data/postgres/{ => sql}/my_data.sql (100%) diff --git a/README.md b/README.md index 96d3db9..39c7093 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # insta-infra -Spin up any tool or service straight away on your local laptop. Tells you how to connect to it. +Spin up any service straight away on your local laptop. Tells you how to connect to it. - Simple commands - Add custom data (i.e startup SQL scripts) @@ -49,6 +49,14 @@ mysql mysql:3306 localhost:3306 host.docker.internal:3306 ./run.sh list ``` +### Remove persisted data + +```shell +./run.sh [remove|-r] +./run.sh -r #remove all service persisted data +./run.sh remove postgres +``` + ### Run from anywhere In your `.bashrc, .zshrc, ...`, add: @@ -64,11 +72,18 @@ insta -l insta postgres insta -c postgres insta -d +insta -r postgres ``` ### Custom data Alter data in [`data`](data) folder. + +You may notice that for some services (such as Cassandra, Postgres, MySQL), they follow the same pattern for custom +data. They have a `sql` directory which contains data files with DDL statements and an `init.sh` script that will help +execute them at startup. This allows you to dump all your `.sql` files into the directory, and it will be automatically +run at startup. + ### Persisted data @@ -76,6 +91,20 @@ If any data is persisted from the services to carry across sessions, it gets pus `./data//persist` +### Authentication + +By default, users and passwords follow what is default in the service. For those services where the user and password +can be altered at startup, it can be altered using environment variable pattern: +```shell +_USER=... +_PASSWORD=... +``` + +For example: +```shell +POSTGRES_USER=my-user POSTGRES_PASSWORD=my-password ./run.sh postgres +``` + ## Services | Service Type | Service | Supported | diff --git a/data/cassandra/my_data.cql b/data/cassandra/cql/my_data.cql similarity index 100% rename from data/cassandra/my_data.cql rename to data/cassandra/cql/my_data.cql diff --git a/data/cassandra/init.sh b/data/cassandra/init.sh new file mode 100755 index 0000000..ceb7651 --- /dev/null +++ b/data/cassandra/init.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +count=0 +total=0 + +for f in $(ls /tmp/data/*.cql); +do + total=$((total + 1)) + cqlsh -u "${CASSANDRA_USER:-cassandra}" -p "${CASSANDRA_PASSWORD:-cassandra}" -f "${f}" cassandra 9042 + if [[ $? -eq 0 ]]; + then + count=$((count + 1)) + fi; +done; + +echo "Executed ${count} out of ${total} files" \ No newline at end of file diff --git a/data/clickhouse/docker-entrypoint-initdb.d/1_create_postgres_connection.sh b/data/clickhouse/docker-entrypoint-initdb.d/1_create_postgres_connection.sh deleted file mode 100755 index a3b5457..0000000 --- a/data/clickhouse/docker-entrypoint-initdb.d/1_create_postgres_connection.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e -clickhouse client -n <<-EOSQL -# EXPERIMENTAL materialized postgres database https://clickhouse.com/docs/en/integrations/postgresql#using-the-materializedpostgresql-database-engine -SET allow_experimental_database_materialized_postgresql=1; - -SET allow_experimental_materialized_postgresql_table=1; - -CREATE DATABASE customer_postgres_db ENGINE = PostgreSQL('postgres:5432', 'customer', 'postgres', 'postgres', 'account', 0); -EOSQL \ No newline at end of file diff --git a/data/clickhouse/init.sh b/data/clickhouse/init.sh new file mode 100755 index 0000000..54fb8ca --- /dev/null +++ b/data/clickhouse/init.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +count=0 +total=0 + +for f in $(ls /tmp/data/*.sql); +do + clickhouse client --host clickhouse-server --multiquery < "$f" + if [[ $? -eq 0 ]]; + then + count=$((count + 1)) + fi; +done; + +echo "Executed ${count} out of ${total} files" \ No newline at end of file diff --git a/data/clickhouse/sql/create_postgres_connection.sql b/data/clickhouse/sql/create_postgres_connection.sql new file mode 100644 index 0000000..a983e68 --- /dev/null +++ b/data/clickhouse/sql/create_postgres_connection.sql @@ -0,0 +1,5 @@ +SET allow_experimental_database_materialized_postgresql=1; + +SET allow_experimental_materialized_postgresql_table=1; + +CREATE DATABASE customer_postgres_db ENGINE = PostgreSQL('postgres:5432', 'customer', 'postgres', 'postgres', 'account', 0); \ No newline at end of file diff --git a/data/druid/environment b/data/druid/environment index 4c7a70d..ee02227 100644 --- a/data/druid/environment +++ b/data/druid/environment @@ -34,8 +34,6 @@ druid_zk_service_host=zookeeper druid_metadata_storage_host= druid_metadata_storage_type=postgresql druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid -druid_metadata_storage_connector_user=postgres -druid_metadata_storage_connector_password=postgres druid_coordinator_balancer_strategy=cachingCost diff --git a/data/kafka/init.sh b/data/kafka/init.sh new file mode 100755 index 0000000..369c3ce --- /dev/null +++ b/data/kafka/init.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +host="kafka" +# blocks until kafka is reachable +kafka-topics --bootstrap-server ${host}:29092 --list + +if [[ -z "$KAFKA_TOPICS" ]]; then + echo "No Kafka topics provided via KAFKA_TOPICS environment variable" + exit 0 +fi + +echo "Creating kafka topics" +for t in $(echo "$KAFKA_TOPICS" | sed s/,/\\n/g); do + kafka-topics --create --if-not-exists --topic "${t}" --bootstrap-server ${host}:9092 --replication-factor 1 --partitions 1 +done + +echo "Successfully created the following topics:" +kafka-topics --bootstrap-server ${host}:29092 --list diff --git a/data/kafka/my_data.sh b/data/kafka/my_data.sh deleted file mode 100755 index d3801f9..0000000 --- a/data/kafka/my_data.sh +++ /dev/null @@ -1,11 +0,0 @@ -host="kafka" -# blocks until kafka is reachable -kafka-topics --bootstrap-server $host:29092 --list - -echo 'Creating kafka topics' -kafka-topics --bootstrap-server $host:29092 --create --if-not-exists --topic account-topic --replication-factor 1 --partitions 1 - -echo 'Successfully created the following topics:' -kafka-topics --bootstrap-server $host:29092 --list - -#kafka-topics --delete --topic account-topic --bootstrap-server localhost:9092 diff --git a/data/mysql/init.sh b/data/mysql/init.sh new file mode 100755 index 0000000..924bcbe --- /dev/null +++ b/data/mysql/init.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +count=0 +total=0 + +for f in $(ls /tmp/data/*.sql); +do + total=$((total + 1)) + mysql -u root -p"${MYSQL_PASSWORD:-root}" -h "mysql" < "${f}" + if [[ $? -eq 0 ]]; + then + count=$((count + 1)) + fi; +done; + +echo "Executed ${count} out of ${total} files" \ No newline at end of file diff --git a/data/mysql/my_data.sql b/data/mysql/sql/my_data.sql similarity index 97% rename from data/mysql/my_data.sql rename to data/mysql/sql/my_data.sql index 68e5a3c..87ebaa8 100644 --- a/data/mysql/my_data.sql +++ b/data/mysql/sql/my_data.sql @@ -1,4 +1,4 @@ -CREATE DATABASE customer; +CREATE DATABASE IF NOT EXISTS customer; CREATE TABLE IF NOT EXISTS customer.accounts ( diff --git a/data/postgres/init.sh b/data/postgres/init.sh new file mode 100755 index 0000000..7110c19 --- /dev/null +++ b/data/postgres/init.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +count=0 +total=0 + +for f in $(ls /tmp/data/*.sql); +do + total=$((total + 1)) + psql -U${POSTGRES_USER:-postgres} -hpostgres -f "${f}" + if [[ $? -eq 0 ]]; + then + count=$((count + 1)) + fi; +done; + +echo "Executed ${count} out of ${total} files" \ No newline at end of file diff --git a/data/postgres/my_data.sql b/data/postgres/sql/my_data.sql similarity index 100% rename from data/postgres/my_data.sql rename to data/postgres/sql/my_data.sql diff --git a/docker-compose.yaml b/docker-compose.yaml index 309820e..4d3b3d0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,8 +3,8 @@ "activemq": "container_name": "activemq" "environment": - - "ARTEMIS_USER=artemis" - - "ARTEMIS_PASSWORD=artemis" + - "ARTEMIS_USER=${ARTEMIS_USER:-artemis}" + - "ARTEMIS_PASSWORD=${ARTEMIS_PASSWORD:-artemis}" "healthcheck": "interval": "15s" "retries": 3 @@ -24,7 +24,7 @@ "condition": "service_completed_successfully" "environment": - "AIRFLOW_UID=50000" - - "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres/airflow" + - "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres/airflow" - "AIRFLOW__CORE__FERNET_KEY=" - "AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true" - "AIRFLOW__CORE__LOAD_EXAMPLES=true" @@ -86,20 +86,20 @@ "container_name": "airflow-init" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "entrypoint": "/bin/bash" # yamllint enable rule:line-length "environment": - "AIRFLOW_UID=50000" - - "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres/airflow" + - "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres/airflow" - "_AIRFLOW_DB_MIGRATE=true" - "_AIRFLOW_WWW_USER_CREATE=true" - - "_AIRFLOW_WWW_USER_USERNAME=airflow" - - "_AIRFLOW_WWW_USER_PASSWORD=airflow" + - "_AIRFLOW_WWW_USER_USERNAME=${AIRFLOW_USER:-airflow}" + - "_AIRFLOW_WWW_USER_PASSWORD=${AIRFLOW_PASSWORD:-airflow}" - "_PIP_ADDITIONAL_REQUIREMENTS=" "image": "apache/airflow:${AIRFLOW_VERSION:-2.9.2}" "user": "0:0" - "cassandra": + "cassandra-server": "cap_add": - "IPC_LOCK" "container_name": "cassandra" @@ -117,12 +117,29 @@ "memlock": -1 "volumes": - "./data/cassandra/persist:/var/lib/cassandra" - - "./data/cassandra/my_data.cql:/docker-entrypoint-initdb.d/my_data.cql" - "clickhouse": + "cassandra": + "command": ["-c", "/tmp/scripts/init.sh"] + "container_name": "cassandra-data" + "depends_on": + "cassandra-server": + "condition": "service_healthy" + "entrypoint": "/bin/bash" + "environment": + - "DS_LICENSE=accept" + "image": "datacatering/dse-server:6.8.48" + "volumes": + - "./data/cassandra/init.sh:/tmp/scripts/init.sh" + - "./data/cassandra/cql:/tmp/data" + "clickhouse-server": "container_name": "clickhouse" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" + "healthcheck": + "interval": "10s" + "retries": 3 + "test": "wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1" + "timeout": "5s" "hostname": "clickhouse" "image": "clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-24.5.3}" "ports": @@ -130,7 +147,19 @@ - "9000:9000" "user": "101:101" "volumes": - - "./data/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d" + - "./data/clickhouse/persist:/var/lib/clickhouse" + "clickhouse": + "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] + "container_name": "clickhouse-data" + "depends_on": + "clickhouse-server": + "condition": "service_healthy" + "hostname": "clickhouse" + "image": "clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-24.5.3}" + "user": "101:101" + "volumes": + - "./data/clickhouse/init.sh:/tmp/scripts/init.sh" + - "./data/clickhouse/sql:/tmp/data" "cockroachdb": "command": ["start-single-node", "--insecure"] "container_name": "cockroachdb" @@ -149,12 +178,12 @@ "container_name": "dagster" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "entrypoint": ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000", "-w", "/opt/dagster/app/workspace.yaml"] "environment": - "DAGSTER_POSTGRES_HOST=postgres" - - "DAGSTER_POSTGRES_USER=postgres" - - "DAGSTER_POSTGRES_PASSWORD=postgres" + - "DAGSTER_POSTGRES_USER=${POSTGRES_USER:-postgres}" + - "DAGSTER_POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}" - "DAGSTER_POSTGRES_DB=dagster" - "DAGSTER_HOME=/opt/dagster/dagster_home/" "image": "dagster/dagster-k8s:${DAGSTER_VERSION:-1.7.10}" @@ -167,7 +196,7 @@ "container_name": "data-caterer" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "environment": - "DEPLOY_MODE=standalone" "image": "datacatering/data-caterer-basic:${DATA_CATERER_VERSION:-0.10.10}" @@ -217,7 +246,7 @@ "container_name": "doris" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "image": "apache/doris:${DORIS_VERSION:-doris-all-in-one-2.1.0}" "ports": - "8030:8030" @@ -239,6 +268,9 @@ "condition": "service_healthy" "zookeeper": "condition": "service_healthy" + "environment": + - "druid_metadata_storage_connector_user=${POSTGRES_USER:-postgres}" + - "druid_metadata_storage_connector_password=${POSTGRES_PASSWORD:-postgres}" "env_file": "data/druid/environment" "healthcheck": "interval": "10s" @@ -260,6 +292,9 @@ "condition": "service_healthy" "zookeeper": "condition": "service_healthy" + "environment": + - "druid_metadata_storage_connector_user=${POSTGRES_USER:-postgres}" + - "druid_metadata_storage_connector_password=${POSTGRES_PASSWORD:-postgres}" "env_file": "data/druid/environment" "healthcheck": "interval": "10s" @@ -276,9 +311,12 @@ "container_name": "druid-coordinator" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "zookeeper": "condition": "service_healthy" + "environment": + - "druid_metadata_storage_connector_user=${POSTGRES_USER:-postgres}" + - "druid_metadata_storage_connector_password=${POSTGRES_PASSWORD:-postgres}" "env_file": "data/druid/environment" "healthcheck": "interval": "10s" @@ -301,6 +339,9 @@ "condition": "service_healthy" "zookeeper": "condition": "service_healthy" + "environment": + - "druid_metadata_storage_connector_user=${POSTGRES_USER:-postgres}" + - "druid_metadata_storage_connector_password=${POSTGRES_PASSWORD:-postgres}" "env_file": "data/druid/environment" "healthcheck": "interval": "10s" @@ -323,6 +364,9 @@ "condition": "service_healthy" "zookeeper": "condition": "service_healthy" + "environment": + - "druid_metadata_storage_connector_user=${POSTGRES_USER:-postgres}" + - "druid_metadata_storage_connector_password=${POSTGRES_PASSWORD:-postgres}" "env_file": "data/druid/environment" "healthcheck": "interval": "10s" @@ -340,7 +384,7 @@ "container_name": "duckdb" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "entrypoint": ["tail", "-F", "anything"] "image": "datacatering/duckdb:${DUCKDB_VERSION:-v1.0.0}" "volumes": @@ -350,7 +394,7 @@ "environment": - "node.name=elasticsearch" - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - "ELASTIC_PASSWORD=elasticsearch" + - "ELASTIC_PASSWORD=${ELASTICSEARCH_PASSWORD:-elasticsearch}" - "discovery.type=single-node" "image": "docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.14.1}" "ports": @@ -368,13 +412,13 @@ - "sqlite" "environment": - "TLS_ENABLED=1" - - "FLIGHT_PASSWORD=flight_password" + - "FLIGHT_PASSWORD=${FLIGHT_SQL_PASSWORD:-flight_password}" - "PRINT_QUERIES=1" "image": "voltrondata/flight-sql:${FLIGHT_SQL_VERSION:-v1.4.0}" "ports": - "31337:31337" "volumes": - - "./data/flight-sql:/opt/data" + - "./data/flight-sql/persist:/opt/data" "flink": "command": "taskmanager" "container_name": "flink" @@ -411,10 +455,12 @@ "depends_on": "kafka-server": "condition": "service_healthy" - "entrypoint": ["/bin/sh", "-c", "/opt/app/my_data.sh"] + "entrypoint": ["/bin/sh", "-c", "/tmp/scripts/init.sh"] + "environment": + - "KAFKA_TOPICS=${KAFKA_TOPICS:-accounts,transactions}" "image": "confluentinc/confluent-local:${KAFKA_VERSION:-7.6.1}" "volumes": - - "./data/kafka:/opt/app" + - "./data/kafka/init.sh:/tmp/scripts/init.sh" "kafka-server": "container_name": "kafka" "environment": @@ -439,15 +485,15 @@ "container_name": "keycloak" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "environment": - "KC_DB=postgres" - - "KC_DB_USERNAME=postgres" - - "KC_DB_PASSWORD=postgres" + - "KC_DB_USERNAME=${POSTGRES_USER:-postgres}" + - "KC_DB_PASSWORD=${POSTGRES_PASSWORD:-postgres}" - "KC_DB_URL=jdbc:postgresql://postgres:5432/keycloak" - "KC_REALM_NAME=myrealm" - - "KEYCLOAK_ADMIN=admin" - - "KEYCLOAK_ADMIN_PASSWORD=admin" + - "KEYCLOAK_ADMIN=${KEYCLOAK_USER:-admin}" + - "KEYCLOAK_ADMIN_PASSWORD=${KEYCLOAK_PASSWORD:-admin}" "image": "quay.io/keycloak/keycloak:${KEYCLOACK_VERSION:-25.0.0}" "ports": - "8082:8080" @@ -468,8 +514,8 @@ "mariadb": "container_name": "mariadb" "environment": - - "MARIADB_USER=user" - - "MARIADB_PASSWORD=password" + - "MARIADB_USER=${MARIADB_USER:-user}" + - "MARIADB_PASSWORD=${MARIADB_PASSWORD:-password}" - "MARIADB_ROOT_PASSWORD=root" - "MARIADB_DATABASE=customer" "image": "mariadb:${MARIADB_VERSION:-11.4.2}" @@ -478,7 +524,6 @@ "restart": "always" "volumes": - "./data/mariadb/persist:/var/lib/mysql:Z" - - "./data/mariadb:/opt/app" "marquez": "container_name": "marquez-web" "depends_on": @@ -490,12 +535,11 @@ "ports": - "3001:3000" "marquez-data": - "command": ["-c", "/opt/app/seed.sh"] + "command": ["/bin/bash", "-c", "/opt/app/seed.sh"] "container_name": "marquez-data" "depends_on": "marquez-server": "condition": "service_healthy" - "entrypoint": "/bin/bash" "environment": - "MARQUEZ_URL=http://marquez:5000" "image": "marquezproject/marquez:${MARQUEZ_VERSION:-0.47.0}" @@ -505,7 +549,7 @@ "container_name": "marquez" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "environment": - "MARQUEZ_CONFIG=/opt/app/marquez.yaml" - "MARQUEZ_PORT=5000" @@ -513,8 +557,8 @@ - "POSTGRES_HOST=postgres" - "POSTGRES_PORT=5432" - "POSTGRES_DB=marquez" - - "POSTGRES_USER=postgres" - - "POSTGRES_PASSWORD=postgres" + - "POSTGRES_USER=${POSTGRES_USER:-postgres}" + - "POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}" "healthcheck": "interval": "10s" "retries": 3 @@ -531,8 +575,8 @@ "command": ["server", "/data", "--console-address", ":9001"] "container_name": "minio" "environment": - - "MINIO_ROOT_USER=minioadmin" - - "MINIO_ROOT_PASSWORD=minioadmin" + - "MINIO_ROOT_USER=${MINIO_USER:-minioadmin}" + - "MINIO_ROOT_PASSWORD=${MINIO_PASSWORD:-minioadmin}" "healthcheck": "interval": "5s" "retries": 3 @@ -550,35 +594,47 @@ "depends_on": - "mongodb-server" "environment": - - "CONN_STR=mongodb://user:password@mongodb-server" + - "CONN_STR=mongodb://${MONGODB_USER:-user}:${MONGODB_PASSWORD:-password}@mongodb-server" "image": "mongodb/mongodb-community-server:${MONGODB_VERSION:-7.0.11-ubi8}" "volumes": - "./data/mongodb:/opt/app" "mongodb-server": "container_name": "mongodb" "environment": - - "MONGO_INITDB_ROOT_USERNAME=user" - - "MONGO_INITDB_ROOT_PASSWORD=password" + - "MONGO_INITDB_ROOT_USERNAME=${MONGODB_USER:-user}" + - "MONGO_INITDB_ROOT_PASSWORD=${MONGODB_PASSWORD:-password}" "image": "mongo:${MONGODB_VERSION:-7.0.11}" "ports": - "27017:27017" "volumes": - "./data/mongodb/persist:/data/db" - "mysql": + "mysql-server": "command": "--mysql-native-password=ON" "container_name": "mysql" "environment": - - "MYSQL_ROOT_PASSWORD=root" + - "MYSQL_ROOT_PASSWORD=${MYSQL_PASSWORD:-root}" "healthcheck": "retries": 3 - "test": ["CMD-SHELL", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-p$$MYSQL_ROOT_PASSWORD"] - "timeout": "20s" + "test": ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-p$$MYSQL_ROOT_PASSWORD"] + "timeout": "5s" + "interval": "5s" "image": "mysql:${MYSQL_VERSION:-8.4.0}" "ports": - "3306:3306" "volumes": - "./data/mysql/persist:/var/lib/mysql" - - "./data/mysql/my_data.sql:/docker-entrypoint-initdb.d/my_data.sql" + "mysql": + "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] + "container_name": "mysql-data" + "depends_on": + "mysql-server": + "condition": "service_healthy" + "environment": + - "MYSQL_PASSWORD=${MYSQL_PASSWORD:-root}" + "image": "mysql:${MYSQL_VERSION:-8.4.0}" + "volumes": + - "./data/mysql/init.sh:/tmp/scripts/init.sh" + - "./data/mysql/sql:/tmp/data" "neo4j": "container_name": "neo4j" "environment": @@ -647,11 +703,11 @@ "ports": - "9000:9000" "restart": "unless-stopped" - "postgres": + "postgres-server": "container_name": "postgres" "environment": - - "POSTGRES_USER=postgres" - - "POSTGRES_PASSWORD=postgres" + - "POSTGRES_USER=${POSTGRES_USER:-postgres}" + - "POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}" - "PGDATA=/data/postgres" "healthcheck": "interval": "10s" @@ -663,7 +719,19 @@ - "5432:5432" "volumes": - "./data/postgres/persist:/data/postgres" - - "./data/postgres/my_data.sql:/docker-entrypoint-initdb.d/my_data.sql" + "postgres": + "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] + "container_name": "postgres-data" + "depends_on": + "postgres-server": + "condition": "service_healthy" + "environment": + - "POSTGRES_USER=${POSTGRES_USER:-postgres}" + - "PGPASSWORD=${POSTGRES_PASSWORD:-postgres}" + "image": "postgres:${POSTGRES_VERSION:-16.3}" + "volumes": + - "./data/postgres/init.sh:/tmp/scripts/init.sh" + - "./data/postgres/sql:/tmp/data" "prefect": "container_name": "prefect-data" "depends_on": @@ -680,13 +748,13 @@ "container_name": "prefect" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "entrypoint": ["/opt/prefect/entrypoint.sh", "prefect", "server", "start"] "environment": - "PREFECT_UI_URL=http://127.0.0.1:4200/api" - "PREFECT_API_URL=http://127.0.0.1:4200/api" - "PREFECT_SERVER_API_HOST=0.0.0.0" - - "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://postgres:postgres@postgres:5432/prefect" + - "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres:5432/prefect" "image": "prefecthq/prefect:${PREFECT_VERSION:-2.19.5-python3.11}" "ports": - "4200:4200" @@ -697,7 +765,7 @@ "container_name": "presto" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "image": "prestodb/presto:${PRESTO_VERSION:-0.287}" "ports": - "8083:8080" @@ -707,8 +775,8 @@ "rabbitmq": "container_name": "rabbitmq" "environment": - - "RABBITMQ_DEFAULT_USER=guest" - - "RABBITMQ_DEFAULT_PASS=guest" + - "RABBITMQ_DEFAULT_USER=${RABBITMQ_USER:-guest}" + - "RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD:-guest}" "healthcheck": "interval": "30s" "retries": 3 @@ -738,7 +806,7 @@ "max_attempts": 1 "environment": - "username_admin_globalaccesslevel=admin" - - "username_admin_password=admin" + - "username_admin_password=${SOLACE_PASSWORD:-admin}" - "system_scaling_maxconnectioncount=100" "healthcheck": "interval": "30s" @@ -788,7 +856,7 @@ "container_name": "trino" "depends_on": "postgres": - "condition": "service_healthy" + "condition": "service_completed_successfully" "image": "trinodb/trino:${TRINO_VERSION:-449}" "ports": - "8084:8080" diff --git a/run.sh b/run.sh index 1dd6aec..2ee15d3 100755 --- a/run.sh +++ b/run.sh @@ -9,19 +9,19 @@ NC='\033[0m' SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) connection_commands=" -activemq='/var/lib/artemis-instance/bin/artemis shell --user artemis --password artemis' +activemq='/var/lib/artemis-instance/bin/artemis shell --user ${ARTEMIS_USER:-artemis} --password ${ARTEMIS_PASSWORD:-artemis}' cassandra='cqlsh' clickhouse='clickhouse-client' cockroachdb='./cockroach sql --insecure' doris='mysql -uroot -P9030 -h127.0.0.1' duckdb='./duckdb' -elasticsearch='elasticsearch-sql-cli http://elastic:elasticsearch@localhost:9200' -flight-sql='flight_sql_client --command Execute --host localhost --port 31337 --username flight_username --password flight_password --query 'SELECT version()' --use-tls --tls-skip-verify' -mariadb='mariadb --user=user --password=password' -mongodb-connect='mongosh mongodb://root:root@mongodb' -mysql='mysql -u root -proot' +elasticsearch='elasticsearch-sql-cli http://elastic:${ELASTICSEARCH_PASSWORD:-elasticsearch}@localhost:9200' +flight-sql='flight_sql_client --command Execute --host localhost --port 31337 --username ${FLIGHT_SQL_USER:-flight_username} --password ${FLIGHT_SQL_PASSWORD:-flight_password} --query 'SELECT version()' --use-tls --tls-skip-verify' +mariadb='mariadb --user=${MARIADB_USER:-user} --password=${MARIADB_PASSWORD:-password}' +mongodb-connect='mongosh mongodb://${MONGODB_USER:-root}:${MONGODB_PASSWORD:-root}@mongodb' +mysql='mysql -u ${MYSQL_USER:-root} -p${MYSQL_PASSWORD:-root}' neo4j='cypher-shell -u neo4j -p test' -postgres='PGPASSWORD=postgres psql -Upostgres' +postgres='PGPASSWORD=${POSTGRES_PASSWORD:-postgres} psql -U${POSTGRES_USER:-postgres}' prefect-data='bash' presto='presto-cli' trino='trino' @@ -33,17 +33,19 @@ flink-jobmanager='bash' usage() { echo "Usage: $(basename "$0") [options...] [services...]" echo - echo " Name of services to run" - echo " -c, connect [service] Connect to service" - echo " -d, down [services...] Shutdown services (if empty, shutdown all services)" - echo " -h, --help Show help" - echo " -l, list List supported services" + echo " Name of services to run" + echo " -c, connect [service] Connect to service" + echo " -d, down [services...] Shutdown services (if empty, shutdown all services)" + echo " -h, --help, help Show help" + echo " -l, list List supported services" + echo " -r, remove [services...] Remove persisted data (if empty, remove all services persisted data)" echo echo "Examples:" echo " $(basename "$0") -l" - echo " $(basename "$0") postgres Spin up Postgres" - echo " $(basename "$0") -c postgres Connect to Postgres" - echo " $(basename "$0") -d Bring Postgres down" + echo " $(basename "$0") postgres Spin up Postgres" + echo " $(basename "$0") -c postgres Connect to Postgres" + echo " $(basename "$0") -d Bring Postgres down" + echo " $(basename "$0") -r postgres Remove Postgres persisted data" exit 0 } @@ -123,8 +125,30 @@ log_how_to_connect() { done | column -t -s ',' } +remove_persisted_data() { + if [ -z "$1" ]; then + read -p "Continue to remove all persisted data? (Y/n)" CONT + if [ "$CONT" = "Y" ]; then + echo "Removing all services persisted data..." + find "${SCRIPT_DIR}/data" -type d -name "persist" -maxdepth 2 -exec rm -r {} \; + else + echo "Not removing any persisted data"; + fi + else + read -p "Continue to remove persisted data for services: $*? (Y/n)" CONT + if [ "$CONT" = "Y" ]; then + echo "Removing persisted data for services: $*..." + for service in "$@"; do + rm -r "${SCRIPT_DIR}/data/${service}/persist" + done + else + echo "Not removing any persisted data"; + fi + fi +} + case $1 in - "--help"|"-h") + "-h"|"--help"|"help") usage ;; "-c"|"connect") @@ -136,6 +160,9 @@ case $1 in "-l"|"list") list_supported_services ;; + "-r"|"remove") + remove_persisted_data "${@:2}" + ;; *) if [ $# -eq 0 ]; then usage From a766ce129a857a6936f85bbb7c6f4d2fe5fb1b97 Mon Sep 17 00:00:00 2001 From: pflooky Date: Thu, 20 Jun 2024 04:56:36 +0000 Subject: [PATCH 2/2] Format docker-compose.yaml --- docker-compose.yaml | 106 ++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 4d3b3d0..ad04666 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -99,6 +99,19 @@ - "_PIP_ADDITIONAL_REQUIREMENTS=" "image": "apache/airflow:${AIRFLOW_VERSION:-2.9.2}" "user": "0:0" + "cassandra": + "command": ["-c", "/tmp/scripts/init.sh"] + "container_name": "cassandra-data" + "depends_on": + "cassandra-server": + "condition": "service_healthy" + "entrypoint": "/bin/bash" + "environment": + - "DS_LICENSE=accept" + "image": "datacatering/dse-server:6.8.48" + "volumes": + - "./data/cassandra/init.sh:/tmp/scripts/init.sh" + - "./data/cassandra/cql:/tmp/data" "cassandra-server": "cap_add": - "IPC_LOCK" @@ -117,19 +130,18 @@ "memlock": -1 "volumes": - "./data/cassandra/persist:/var/lib/cassandra" - "cassandra": - "command": ["-c", "/tmp/scripts/init.sh"] - "container_name": "cassandra-data" + "clickhouse": + "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] + "container_name": "clickhouse-data" "depends_on": - "cassandra-server": + "clickhouse-server": "condition": "service_healthy" - "entrypoint": "/bin/bash" - "environment": - - "DS_LICENSE=accept" - "image": "datacatering/dse-server:6.8.48" + "hostname": "clickhouse" + "image": "clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-24.5.3}" + "user": "101:101" "volumes": - - "./data/cassandra/init.sh:/tmp/scripts/init.sh" - - "./data/cassandra/cql:/tmp/data" + - "./data/clickhouse/init.sh:/tmp/scripts/init.sh" + - "./data/clickhouse/sql:/tmp/data" "clickhouse-server": "container_name": "clickhouse" "depends_on": @@ -148,18 +160,6 @@ "user": "101:101" "volumes": - "./data/clickhouse/persist:/var/lib/clickhouse" - "clickhouse": - "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] - "container_name": "clickhouse-data" - "depends_on": - "clickhouse-server": - "condition": "service_healthy" - "hostname": "clickhouse" - "image": "clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-24.5.3}" - "user": "101:101" - "volumes": - - "./data/clickhouse/init.sh:/tmp/scripts/init.sh" - - "./data/clickhouse/sql:/tmp/data" "cockroachdb": "command": ["start-single-node", "--insecure"] "container_name": "cockroachdb" @@ -268,10 +268,10 @@ "condition": "service_healthy" "zookeeper": "condition": "service_healthy" + "env_file": "data/druid/environment" "environment": - "druid_metadata_storage_connector_user=${POSTGRES_USER:-postgres}" - "druid_metadata_storage_connector_password=${POSTGRES_PASSWORD:-postgres}" - "env_file": "data/druid/environment" "healthcheck": "interval": "10s" "retries": 3 @@ -292,10 +292,10 @@ "condition": "service_healthy" "zookeeper": "condition": "service_healthy" + "env_file": "data/druid/environment" "environment": - "druid_metadata_storage_connector_user=${POSTGRES_USER:-postgres}" - "druid_metadata_storage_connector_password=${POSTGRES_PASSWORD:-postgres}" - "env_file": "data/druid/environment" "healthcheck": "interval": "10s" "retries": 3 @@ -314,10 +314,10 @@ "condition": "service_completed_successfully" "zookeeper": "condition": "service_healthy" + "env_file": "data/druid/environment" "environment": - "druid_metadata_storage_connector_user=${POSTGRES_USER:-postgres}" - "druid_metadata_storage_connector_password=${POSTGRES_PASSWORD:-postgres}" - "env_file": "data/druid/environment" "healthcheck": "interval": "10s" "retries": 3 @@ -339,10 +339,10 @@ "condition": "service_healthy" "zookeeper": "condition": "service_healthy" + "env_file": "data/druid/environment" "environment": - "druid_metadata_storage_connector_user=${POSTGRES_USER:-postgres}" - "druid_metadata_storage_connector_password=${POSTGRES_PASSWORD:-postgres}" - "env_file": "data/druid/environment" "healthcheck": "interval": "10s" "retries": 3 @@ -364,10 +364,10 @@ "condition": "service_healthy" "zookeeper": "condition": "service_healthy" + "env_file": "data/druid/environment" "environment": - "druid_metadata_storage_connector_user=${POSTGRES_USER:-postgres}" - "druid_metadata_storage_connector_password=${POSTGRES_PASSWORD:-postgres}" - "env_file": "data/druid/environment" "healthcheck": "interval": "10s" "retries": 3 @@ -608,33 +608,33 @@ - "27017:27017" "volumes": - "./data/mongodb/persist:/data/db" + "mysql": + "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] + "container_name": "mysql-data" + "depends_on": + "mysql-server": + "condition": "service_healthy" + "environment": + - "MYSQL_PASSWORD=${MYSQL_PASSWORD:-root}" + "image": "mysql:${MYSQL_VERSION:-8.4.0}" + "volumes": + - "./data/mysql/init.sh:/tmp/scripts/init.sh" + - "./data/mysql/sql:/tmp/data" "mysql-server": "command": "--mysql-native-password=ON" "container_name": "mysql" "environment": - "MYSQL_ROOT_PASSWORD=${MYSQL_PASSWORD:-root}" "healthcheck": + "interval": "5s" "retries": 3 "test": ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-p$$MYSQL_ROOT_PASSWORD"] "timeout": "5s" - "interval": "5s" "image": "mysql:${MYSQL_VERSION:-8.4.0}" "ports": - "3306:3306" "volumes": - "./data/mysql/persist:/var/lib/mysql" - "mysql": - "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] - "container_name": "mysql-data" - "depends_on": - "mysql-server": - "condition": "service_healthy" - "environment": - - "MYSQL_PASSWORD=${MYSQL_PASSWORD:-root}" - "image": "mysql:${MYSQL_VERSION:-8.4.0}" - "volumes": - - "./data/mysql/init.sh:/tmp/scripts/init.sh" - - "./data/mysql/sql:/tmp/data" "neo4j": "container_name": "neo4j" "environment": @@ -703,6 +703,19 @@ "ports": - "9000:9000" "restart": "unless-stopped" + "postgres": + "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] + "container_name": "postgres-data" + "depends_on": + "postgres-server": + "condition": "service_healthy" + "environment": + - "POSTGRES_USER=${POSTGRES_USER:-postgres}" + - "PGPASSWORD=${POSTGRES_PASSWORD:-postgres}" + "image": "postgres:${POSTGRES_VERSION:-16.3}" + "volumes": + - "./data/postgres/init.sh:/tmp/scripts/init.sh" + - "./data/postgres/sql:/tmp/data" "postgres-server": "container_name": "postgres" "environment": @@ -719,19 +732,6 @@ - "5432:5432" "volumes": - "./data/postgres/persist:/data/postgres" - "postgres": - "command": ["/bin/bash", "-c", "/tmp/scripts/init.sh"] - "container_name": "postgres-data" - "depends_on": - "postgres-server": - "condition": "service_healthy" - "environment": - - "POSTGRES_USER=${POSTGRES_USER:-postgres}" - - "PGPASSWORD=${POSTGRES_PASSWORD:-postgres}" - "image": "postgres:${POSTGRES_VERSION:-16.3}" - "volumes": - - "./data/postgres/init.sh:/tmp/scripts/init.sh" - - "./data/postgres/sql:/tmp/data" "prefect": "container_name": "prefect-data" "depends_on":