From a51d14e3ef4b0d8dd8bb154fc5ee1718840fe53f Mon Sep 17 00:00:00 2001 From: Flook Peter Date: Thu, 20 Jun 2024 14:57:00 +0800 Subject: [PATCH] Use inner data folder to contain all data files --- README.md | 2 +- data/cassandra/{cql => data}/my_data.cql | 0 .../{sql => data}/create_postgres_connection.sql | 0 data/marquez/{ => conf}/marquez.yaml | 0 data/marquez/{ => data}/metadata.json | 0 data/marquez/{seed.sh => init.sh} | 4 ++-- data/mysql/{sql => data}/my_data.sql | 0 data/postgres/{sql => data}/my_data.sql | 0 docker-compose.yaml | 16 +++++++++------- 9 files changed, 12 insertions(+), 10 deletions(-) rename data/cassandra/{cql => data}/my_data.cql (100%) rename data/clickhouse/{sql => data}/create_postgres_connection.sql (100%) rename data/marquez/{ => conf}/marquez.yaml (100%) rename data/marquez/{ => data}/metadata.json (100%) rename data/marquez/{seed.sh => init.sh} (68%) rename data/mysql/{sql => data}/my_data.sql (100%) rename data/postgres/{sql => data}/my_data.sql (100%) diff --git a/README.md b/README.md index 39c7093..62d233d 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ insta -r postgres Alter data in [`data`](data) folder. You may notice that for some services (such as Cassandra, Postgres, MySQL), they follow the same pattern for custom -data. They have a `sql` directory which contains data files with DDL statements and an `init.sh` script that will help +data. They have a `data` directory which contains data files with DDL statements and an `init.sh` script that will help execute them at startup. This allows you to dump all your `.sql` files into the directory, and it will be automatically run at startup. diff --git a/data/cassandra/cql/my_data.cql b/data/cassandra/data/my_data.cql similarity index 100% rename from data/cassandra/cql/my_data.cql rename to data/cassandra/data/my_data.cql diff --git a/data/clickhouse/sql/create_postgres_connection.sql b/data/clickhouse/data/create_postgres_connection.sql similarity index 100% rename from data/clickhouse/sql/create_postgres_connection.sql rename to data/clickhouse/data/create_postgres_connection.sql diff --git a/data/marquez/marquez.yaml b/data/marquez/conf/marquez.yaml similarity index 100% rename from data/marquez/marquez.yaml rename to data/marquez/conf/marquez.yaml diff --git a/data/marquez/metadata.json b/data/marquez/data/metadata.json similarity index 100% rename from data/marquez/metadata.json rename to data/marquez/data/metadata.json diff --git a/data/marquez/seed.sh b/data/marquez/init.sh similarity index 68% rename from data/marquez/seed.sh rename to data/marquez/init.sh index 60d3c95..cbe846f 100755 --- a/data/marquez/seed.sh +++ b/data/marquez/init.sh @@ -3,8 +3,8 @@ # Copyright 2018-2023 contributors to the Marquez project # SPDX-License-Identifier: Apache-2.0 # -# Usage: $ ./seed.sh +# Usage: $ ./init.sh set -e -java -jar /usr/src/app/marquez-api-*.jar seed --url "${MARQUEZ_URL:-http://localhost:5000}" --metadata /opt/app/metadata.json +java -jar /usr/src/app/marquez-api-*.jar seed --url "${MARQUEZ_URL:-http://localhost:5000}" --metadata /tmp/data/metadata.json diff --git a/data/mysql/sql/my_data.sql b/data/mysql/data/my_data.sql similarity index 100% rename from data/mysql/sql/my_data.sql rename to data/mysql/data/my_data.sql diff --git a/data/postgres/sql/my_data.sql b/data/postgres/data/my_data.sql similarity index 100% rename from data/postgres/sql/my_data.sql rename to data/postgres/data/my_data.sql diff --git a/docker-compose.yaml b/docker-compose.yaml index 4d3b3d0..8241d4b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -129,7 +129,7 @@ "image": "datacatering/dse-server:6.8.48" "volumes": - "./data/cassandra/init.sh:/tmp/scripts/init.sh" - - "./data/cassandra/cql:/tmp/data" + - "./data/cassandra/data:/tmp/data" "clickhouse-server": "container_name": "clickhouse" "depends_on": @@ -159,7 +159,7 @@ "user": "101:101" "volumes": - "./data/clickhouse/init.sh:/tmp/scripts/init.sh" - - "./data/clickhouse/sql:/tmp/data" + - "./data/clickhouse/data:/tmp/data" "cockroachdb": "command": ["start-single-node", "--insecure"] "container_name": "cockroachdb" @@ -535,16 +535,18 @@ "ports": - "3001:3000" "marquez-data": - "command": ["/bin/bash", "-c", "/opt/app/seed.sh"] + "command": ["-c", "/tmp/scripts/init.sh"] "container_name": "marquez-data" "depends_on": "marquez-server": "condition": "service_healthy" + "entrypoint": "/bin/bash" "environment": - "MARQUEZ_URL=http://marquez:5000" "image": "marquezproject/marquez:${MARQUEZ_VERSION:-0.47.0}" "volumes": - - "./data/marquez:/opt/app" + - "./data/marquez/init.sh:/tmp/scripts/init.sh" + - "./data/marquez/data:/tmp/data" "marquez-server": "container_name": "marquez" "depends_on": @@ -570,7 +572,7 @@ - "5001:5001" "volumes": - "./data/marquez/persist:/opt/marquez" - - "./data/marquez:/opt/app" + - "./data/marquez/conf:/opt/app" "minio": "command": ["server", "/data", "--console-address", ":9001"] "container_name": "minio" @@ -634,7 +636,7 @@ "image": "mysql:${MYSQL_VERSION:-8.4.0}" "volumes": - "./data/mysql/init.sh:/tmp/scripts/init.sh" - - "./data/mysql/sql:/tmp/data" + - "./data/mysql/data:/tmp/data" "neo4j": "container_name": "neo4j" "environment": @@ -731,7 +733,7 @@ "image": "postgres:${POSTGRES_VERSION:-16.3}" "volumes": - "./data/postgres/init.sh:/tmp/scripts/init.sh" - - "./data/postgres/sql:/tmp/data" + - "./data/postgres/data:/tmp/data" "prefect": "container_name": "prefect-data" "depends_on":