Merge pull request #43 from data-catering/remove-persist

Add in remove persisted data command

README.md

@@ -1,6 +1,6 @@
 # insta-infra
 
-Spin up any tool or service straight away on your local laptop. Tells you how to connect to it.
+Spin up any service straight away on your local laptop. Tells you how to connect to it.
 
 - Simple commands
 - Add custom data (i.e startup SQL scripts)
@@ -49,6 +49,14 @@ mysql     mysql:3306              localhost:3306     host.docker.internal:3306
 ./run.sh list
 ```
 
+### Remove persisted data
+
+```shell
+./run.sh [remove|-r] <services>
+./run.sh -r #remove all service persisted data
+./run.sh remove postgres
+```
+
 ### Run from anywhere
 
 In your `.bashrc, .zshrc, ...`, add:
@@ -64,18 +72,39 @@ insta -l
 insta postgres
 insta -c postgres
 insta -d
+insta -r postgres
 ```
 
 ### Custom data
 
 Alter data in [`data`](data) folder.
+
+You may notice that for some services (such as Cassandra, Postgres, MySQL), they follow the same pattern for custom
+data. They have a `sql` directory which contains data files with DDL statements and an `init.sh` script that will help
+execute them at startup. This allows you to dump all your `.sql` files into the directory, and it will be automatically
+run at startup.
+
 
 ### Persisted data
 
 If any data is persisted from the services to carry across sessions, it gets pushed to folder:
 
 `./data/<service>/persist`
 
+### Authentication
+
+By default, users and passwords follow what is default in the service. For those services where the user and password
+can be altered at startup, it can be altered using environment variable pattern:
+```shell
+<service>_USER=...
+<service>_PASSWORD=...
+```
+
+For example:
+```shell
+POSTGRES_USER=my-user POSTGRES_PASSWORD=my-password ./run.sh postgres
+```
+
 ## Services
 
 | Service Type                | Service       | Supported |

data/cassandra/init.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+count=0
+total=0
+
+for f in $(ls /tmp/data/*.cql);
+do
+  total=$((total + 1))
+  cqlsh -u "${CASSANDRA_USER:-cassandra}" -p "${CASSANDRA_PASSWORD:-cassandra}" -f "${f}" cassandra 9042
+  if [[ $? -eq 0 ]];
+  then
+    count=$((count + 1))
+  fi;
+done;
+
+echo "Executed ${count} out of ${total} files"

data/clickhouse/init.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+count=0
+total=0
+
+for f in $(ls /tmp/data/*.sql);
+do
+  clickhouse client --host clickhouse-server --multiquery < "$f"
+  if [[ $? -eq 0 ]];
+  then
+    count=$((count + 1))
+  fi;
+done;
+
+echo "Executed ${count} out of ${total} files"

data/clickhouse/sql/create_postgres_connection.sql

@@ -0,0 +1,5 @@
+SET allow_experimental_database_materialized_postgresql=1;
+
+SET allow_experimental_materialized_postgresql_table=1;
+
+CREATE DATABASE customer_postgres_db ENGINE = PostgreSQL('postgres:5432', 'customer', 'postgres', 'postgres', 'account', 0);

data/druid/environment

@@ -34,8 +34,6 @@ druid_zk_service_host=zookeeper
 druid_metadata_storage_host=
 druid_metadata_storage_type=postgresql
 druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
-druid_metadata_storage_connector_user=postgres
-druid_metadata_storage_connector_password=postgres
 
 druid_coordinator_balancer_strategy=cachingCost
 

data/kafka/init.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+host="kafka"
+# blocks until kafka is reachable
+kafka-topics --bootstrap-server ${host}:29092 --list
+
+if [[ -z "$KAFKA_TOPICS" ]]; then
+  echo "No Kafka topics provided via KAFKA_TOPICS environment variable"
+  exit 0
+fi
+
+echo "Creating kafka topics"
+for t in $(echo "$KAFKA_TOPICS" | sed s/,/\\n/g); do
+  kafka-topics --create --if-not-exists --topic "${t}" --bootstrap-server ${host}:9092 --replication-factor 1 --partitions 1
+done
+
+echo "Successfully created the following topics:"
+kafka-topics --bootstrap-server ${host}:29092 --list

data/mysql/init.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+count=0
+total=0
+
+for f in $(ls /tmp/data/*.sql);
+do
+  total=$((total + 1))
+  mysql -u root -p"${MYSQL_PASSWORD:-root}" -h "mysql" < "${f}"
+  if [[ $? -eq 0 ]];
+  then
+    count=$((count + 1))
+  fi;
+done;
+
+echo "Executed ${count} out of ${total} files"

data/mysql/my_data.sql → data/mysql/sql/my_data.sql

@@ -1,4 +1,4 @@
-CREATE DATABASE customer;
+CREATE DATABASE IF NOT EXISTS customer;
 
 CREATE TABLE IF NOT EXISTS customer.accounts
 (

data/postgres/init.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+count=0
+total=0
+
+for f in $(ls /tmp/data/*.sql);
+do
+  total=$((total + 1))
+  psql -U${POSTGRES_USER:-postgres} -hpostgres -f "${f}"
+  if [[ $? -eq 0 ]];
+  then
+    count=$((count + 1))
+  fi;
+done;
+
+echo "Executed ${count} out of ${total} files"