Merge pull request #44 from data-catering/remove-persist

Use inner data folder to contain all data files
data-catering · Jun 20, 2024 · 918e684 · 918e684
2 parents 133cf57 + 3a9a300
commit 918e684
Show file tree

Hide file tree

Showing 9 changed files with 12 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -80,7 +80,7 @@ insta -r postgres
 Alter data in [`data`](data) folder.
 
 You may notice that for some services (such as Cassandra, Postgres, MySQL), they follow the same pattern for custom
-data. They have a `sql` directory which contains data files with DDL statements and an `init.sh` script that will help
+data. They have a `data` directory which contains data files with DDL statements and an `init.sh` script that will help
 execute them at startup. This allows you to dump all your `.sql` files into the directory, and it will be automatically
 run at startup.
 

diff --git a/data/cassandra/cql/my_data.cql → data/cassandra/data/my_data.cql b/data/cassandra/cql/my_data.cql → data/cassandra/data/my_data.cql
diff --git a/...khouse/sql/create_postgres_connection.sql → ...house/data/create_postgres_connection.sql b/...khouse/sql/create_postgres_connection.sql → ...house/data/create_postgres_connection.sql
diff --git a/data/marquez/marquez.yaml → data/marquez/conf/marquez.yaml b/data/marquez/marquez.yaml → data/marquez/conf/marquez.yaml
diff --git a/data/marquez/metadata.json → data/marquez/data/metadata.json b/data/marquez/metadata.json → data/marquez/data/metadata.json
diff --git a/data/marquez/seed.sh → data/marquez/init.sh b/data/marquez/seed.sh → data/marquez/init.sh
@@ -3,8 +3,8 @@
 # Copyright 2018-2023 contributors to the Marquez project
 # SPDX-License-Identifier: Apache-2.0
 #
-# Usage: $ ./seed.sh
+# Usage: $ ./init.sh
 
 set -e
 
-java -jar /usr/src/app/marquez-api-*.jar seed --url "${MARQUEZ_URL:-http://localhost:5000}" --metadata /opt/app/metadata.json
+java -jar /usr/src/app/marquez-api-*.jar seed --url "${MARQUEZ_URL:-http://localhost:5000}" --metadata /tmp/data/metadata.json
diff --git a/data/mysql/sql/my_data.sql → data/mysql/data/my_data.sql b/data/mysql/sql/my_data.sql → data/mysql/data/my_data.sql
diff --git a/data/postgres/sql/my_data.sql → data/postgres/data/my_data.sql b/data/postgres/sql/my_data.sql → data/postgres/data/my_data.sql
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -111,7 +111,7 @@
     "image": "datacatering/dse-server:6.8.48"
     "volumes":
       - "./data/cassandra/init.sh:/tmp/scripts/init.sh"
-      - "./data/cassandra/cql:/tmp/data"
+      - "./data/cassandra/data:/tmp/data"
   "cassandra-server":
     "cap_add":
       - "IPC_LOCK"
@@ -141,7 +141,7 @@
     "user": "101:101"
     "volumes":
       - "./data/clickhouse/init.sh:/tmp/scripts/init.sh"
-      - "./data/clickhouse/sql:/tmp/data"
+      - "./data/clickhouse/data:/tmp/data"
   "clickhouse-server":
     "container_name": "clickhouse"
     "depends_on":
@@ -535,16 +535,18 @@
     "ports":
       - "3001:3000"
   "marquez-data":
-    "command": ["/bin/bash", "-c", "/opt/app/seed.sh"]
+    "command": ["-c", "/tmp/scripts/init.sh"]
     "container_name": "marquez-data"
     "depends_on":
       "marquez-server":
         "condition": "service_healthy"
+    "entrypoint": "/bin/bash"
     "environment":
       - "MARQUEZ_URL=http://marquez:5000"
     "image": "marquezproject/marquez:${MARQUEZ_VERSION:-0.47.0}"
     "volumes":
-      - "./data/marquez:/opt/app"
+      - "./data/marquez/init.sh:/tmp/scripts/init.sh"
+      - "./data/marquez/data:/tmp/data"
   "marquez-server":
     "container_name": "marquez"
     "depends_on":
@@ -570,7 +572,7 @@
       - "5001:5001"
     "volumes":
       - "./data/marquez/persist:/opt/marquez"
-      - "./data/marquez:/opt/app"
+      - "./data/marquez/conf:/opt/app"
   "minio":
     "command": ["server", "/data", "--console-address", ":9001"]
     "container_name": "minio"
@@ -619,7 +621,7 @@
     "image": "mysql:${MYSQL_VERSION:-8.4.0}"
     "volumes":
       - "./data/mysql/init.sh:/tmp/scripts/init.sh"
-      - "./data/mysql/sql:/tmp/data"
+      - "./data/mysql/data:/tmp/data"
   "mysql-server":
     "command": "--mysql-native-password=ON"
     "container_name": "mysql"
@@ -715,7 +717,7 @@
     "image": "postgres:${POSTGRES_VERSION:-16.3}"
     "volumes":
       - "./data/postgres/init.sh:/tmp/scripts/init.sh"
-      - "./data/postgres/sql:/tmp/data"
+      - "./data/postgres/data:/tmp/data"
   "postgres-server":
     "container_name": "postgres"
     "environment":