From 56083c72e887c28e4d522d0c859184f5da901fd1 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Wed, 5 Jun 2024 13:16:06 +0000
Subject: [PATCH 01/25] Update docker.elastic.co/elasticsearch/elasticsearch
 Docker tag to v8.14.0

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 648172b..9ab26f2 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -23,7 +23,7 @@ services:
 
   elasticsearch:
     container_name: "elasticsearch"
-    image: "docker.elastic.co/elasticsearch/elasticsearch:8.13.4"
+    image: "docker.elastic.co/elasticsearch/elasticsearch:8.14.0"
     environment:
       - "node.name=elasticsearch"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"

From 91459f3a04806eea130c86ec291400025467f407 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Thu, 6 Jun 2024 21:30:26 +0000
Subject: [PATCH 02/25] Update dagster/dagster-k8s Docker tag to v1.7.9

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 01c9752..fa6d620 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -279,7 +279,7 @@ services:
 
   dagster:
     container_name: "dagster"
-    image: "dagster/dagster-k8s:1.7.8"
+    image: "dagster/dagster-k8s:1.7.9"
     entrypoint: [ "dagster-webserver", "-h", "0.0.0.0", "-p", "3000", "-w", "/opt/dagster/app/workspace.yaml" ]
     environment:
       - "DAGSTER_POSTGRES_HOST=postgres"

From 8a70434bfe33902737143a13c3e2a3fd60277e19 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Mon, 10 Jun 2024 01:46:38 +0000
Subject: [PATCH 03/25] Update clickhouse/clickhouse-server Docker tag to
 v24.5.1

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 6af7bbc..0dd3781 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -514,7 +514,7 @@ services:
   #real-time OLAP
   clickhouse:
     container_name: "clickhouse"
-    image: "clickhouse/clickhouse-server:24.4.2"
+    image: "clickhouse/clickhouse-server:24.5.1"
     volumes:
       - "./data/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d"
     ports:

From 777ea68bb8dd5f43ddc9192d50a9c6af717e1aaa Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Mon, 10 Jun 2024 01:46:42 +0000
Subject: [PATCH 04/25] Update zookeeper Docker tag to v3.9.2

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 6af7bbc..3761d09 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -163,7 +163,7 @@ services:
   #distributed coordination
   zookeeper:
     container_name: "zookeeper"
-    image: "zookeeper:3.5.10"
+    image: "zookeeper:3.9.2"
     environment:
       - "ZOO_MY_ID=1"
     ports:

From 490e76dbe6ab7a8688ce25bc54a0dbd5c9feb864 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Mon, 10 Jun 2024 09:33:12 +0000
Subject: [PATCH 05/25] Update apache/airflow Docker tag to v2.9.2

---
 docker-compose.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index d1c44ca..45a068a 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -248,7 +248,7 @@ services:
   #job orchestrator
   airflow:
     container_name: "airflow"
-    image: "apache/airflow:2.9.1"
+    image: "apache/airflow:2.9.2"
     command: "standalone"
     environment:
       - "AIRFLOW_UID=50000"
@@ -275,7 +275,7 @@ services:
 
   airflow-init:
     container_name: "airflow-init"
-    image: "apache/airflow:2.9.1"
+    image: "apache/airflow:2.9.2"
     entrypoint: "/bin/bash"
     # yamllint disable rule:line-length
     command:

From ff46ded6c67d514cb28dbd9b2b92226dccce65f2 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Mon, 10 Jun 2024 09:33:17 +0000
Subject: [PATCH 06/25] Update datacatering/data-caterer-basic Docker tag to
 v0.10.8

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index d1c44ca..f350488 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -752,7 +752,7 @@ services:
   #test data management
   data-caterer:
     container_name: "data-caterer"
-    image: "datacatering/data-caterer-basic:0.10.6"
+    image: "datacatering/data-caterer-basic:0.10.8"
     environment:
       - "DEPLOY_MODE=standalone"
     volumes:

From 72c2b34a435c6383d34e75edcd996e06189af912 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Mon, 10 Jun 2024 09:33:21 +0000
Subject: [PATCH 07/25] Update quay.io/keycloak/keycloak Docker tag to v25

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index d1c44ca..54899c7 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -226,7 +226,7 @@ services:
   #identity management
   keycloak:
     container_name: "keycloak"
-    image: "quay.io/keycloak/keycloak:24.0.5"
+    image: "quay.io/keycloak/keycloak:25.0.0"
     command: [ "start-dev", "--import-realm" ]
     environment:
       - "KC_DB=postgres"

From c9cf765303035d3c8db59b8a56ed19f8f16b717b Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Tue, 11 Jun 2024 13:44:00 +0000
Subject: [PATCH 08/25] Update datacatering/data-caterer-basic Docker tag to
 v0.10.10

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index f13d996..95a523b 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -752,7 +752,7 @@ services:
   #test data management
   data-caterer:
     container_name: "data-caterer"
-    image: "datacatering/data-caterer-basic:0.10.8"
+    image: "datacatering/data-caterer-basic:0.10.10"
     environment:
       - "DEPLOY_MODE=standalone"
     volumes:

From 238c7c12992883881a4ced3ad5609c5310dce539 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Wed, 12 Jun 2024 02:15:30 +0000
Subject: [PATCH 09/25] Update neo4j Docker tag to v4.4.34

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index a756e72..4be42ac 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -132,7 +132,7 @@ services:
 
   neo4j:
     container_name: neo4j
-    image: neo4j:4.4.10
+    image: neo4j:4.4.34
     environment:
       - NEO4J_AUTH=none
       - NEO4J_dbms_connector_http_advertised__address=localhost:7474

From 5c59467ff783c79a06c902c3f03a7f0d4e1cc6ff Mon Sep 17 00:00:00 2001
From: Flook Peter <pflooky@hotmail.com>
Date: Wed, 12 Jun 2024 10:20:13 +0800
Subject: [PATCH 10/25] Push data into persist directory

---
 docker-compose.yaml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index b30461d..02c9a26 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -47,14 +47,14 @@ services:
     platform: "linux/amd64"
     command: ["start-single-node", "--insecure"]
     volumes:
-      - "./data/cockroachdb:/cockroach/cockroach-data"
+      - "./data/cockroachdb/persist:/cockroach/cockroach-data"
     ports:
       - "26257:26257"
       - "8080:8080"
     healthcheck:
       test: ["CMD-SHELL", "curl --fail http://localhost:8080/ || exit 1"]
-      interval: 10s
-      timeout: 5s
+      interval: "10s"
+      timeout: "5s"
       retries: 5
 
   cassandra:
@@ -147,21 +147,21 @@ services:
       - "3306:3306"
 
   neo4j:
-    container_name: neo4j
-    image: neo4j:4.4.34
+    container_name: "neo4j"
+    image: "neo4j:4.4.34"
     environment:
-      - NEO4J_AUTH=none
-      - NEO4J_dbms_connector_http_advertised__address=localhost:7474
-      - NEO4J_dbms_connector_bolt_advertised__address=localhost:7687
+      - "NEO4J_AUTH=none"
+      - "NEO4J_dbms_connector_http_advertised__address=localhost:7474"
+      - "NEO4J_dbms_connector_bolt_advertised__address=localhost:7687"
     volumes:
-      - ./data/neo4j:/data
+      - "./data/neo4j/persist:/data"
     ports:
       - "7474:7474"
       - "7687:7687"
     healthcheck:
       test: ["CMD-SHELL", "cypher-shell -u neo4j -p test 'RETURN 1' || exit 1"]
-      interval: 30s
-      timeout: 10s
+      interval: "30s"
+      timeout: "10s"
       retries: 5
 
   postgres:

From 038af19f203b0e6fe9a6d505d92134679db5e7eb Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Wed, 12 Jun 2024 02:20:48 +0000
Subject: [PATCH 11/25] Update neo4j Docker tag to v5

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 02c9a26..969f89e 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -148,7 +148,7 @@ services:
 
   neo4j:
     container_name: "neo4j"
-    image: "neo4j:4.4.34"
+    image: "neo4j:5.20.0"
     environment:
       - "NEO4J_AUTH=none"
       - "NEO4J_dbms_connector_http_advertised__address=localhost:7474"

From 265b607fd58e805cb44bb30aaf581d17a4d7285b Mon Sep 17 00:00:00 2001
From: Flook Peter <pflooky@hotmail.com>
Date: Wed, 12 Jun 2024 10:21:09 +0800
Subject: [PATCH 12/25] Update cockroach to v24.1.0, remove platform

---
 docker-compose.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 02c9a26..02fc8d9 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -43,8 +43,7 @@ services:
   #database
   cockroachdb:
     container_name: "cockroachdb"
-    image: "cockroachdb/cockroach:v22.1.8"
-    platform: "linux/amd64"
+    image: "cockroachdb/cockroach:v24.1.0"
     command: ["start-single-node", "--insecure"]
     volumes:
       - "./data/cockroachdb/persist:/cockroach/cockroach-data"

From f92c55fb2bfd37647d449e9a4681b9aa9a8f7661 Mon Sep 17 00:00:00 2001
From: Flook Peter <pflooky@hotmail.com>
Date: Wed, 12 Jun 2024 10:35:09 +0800
Subject: [PATCH 13/25] Add in example commands to help

---
 run.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/run.sh b/run.sh
index 6201e27..71b47ed 100755
--- a/run.sh
+++ b/run.sh
@@ -33,6 +33,12 @@ usage() {
   echo "    -c, connect [service]   Connect to service"
   echo "    -d, down [services...]  Shutdown services (if empty, shutdown all services)"
   echo "    -l, list                List supported services"
+  echo
+  echo "Examples:"
+  echo "    $(basename "$0") -l"
+  echo "    $(basename "$0") postgres         Spin up Postgres"
+  echo "    $(basename "$0") -c postgres      Connect to Postgres"
+  echo "    $(basename "$0") -d               Bring Postgres down"
   exit 0
 }
 

From 7a36106b874d0acf175f4e714fc8803dcf1f58e9 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Wed, 12 Jun 2024 13:52:00 +0000
Subject: [PATCH 14/25] Update docker.elastic.co/elasticsearch/elasticsearch
 Docker tag to v8.14.1

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 86294c2..b65f48d 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -78,7 +78,7 @@ services:
 
   elasticsearch:
     container_name: "elasticsearch"
-    image: "docker.elastic.co/elasticsearch/elasticsearch:8.14.0"
+    image: "docker.elastic.co/elasticsearch/elasticsearch:8.14.1"
     environment:
       - "node.name=elasticsearch"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"

From 17fabca0832ba4c56d2bd4bcb3b7143cd7a5b13a Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Thu, 13 Jun 2024 20:42:20 +0000
Subject: [PATCH 15/25] Update clickhouse/clickhouse-server Docker tag to
 v24.5.3

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index b65f48d..e53468e 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -587,7 +587,7 @@ services:
   #real-time OLAP
   clickhouse:
     container_name: "clickhouse"
-    image: "clickhouse/clickhouse-server:24.5.1"
+    image: "clickhouse/clickhouse-server:24.5.3"
     volumes:
       - "./data/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d"
     ports:

From 038db19fc5ccb310f06a9d28ca51d799864ad8fa Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Thu, 13 Jun 2024 20:42:23 +0000
Subject: [PATCH 16/25] Update dagster/dagster-k8s Docker tag to v1.7.10

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index b65f48d..14c3ef7 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -366,7 +366,7 @@ services:
 
   dagster:
     container_name: "dagster"
-    image: "dagster/dagster-k8s:1.7.9"
+    image: "dagster/dagster-k8s:1.7.10"
     entrypoint: [ "dagster-webserver", "-h", "0.0.0.0", "-p", "3000", "-w", "/opt/dagster/app/workspace.yaml" ]
     environment:
       - "DAGSTER_POSTGRES_HOST=postgres"

From 6c882505c633e00b3a1a7845f07edb8843cba5fd Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Thu, 13 Jun 2024 20:42:26 +0000
Subject: [PATCH 17/25] Update prefecthq/prefect Docker tag to v2.19.5

---
 docker-compose.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index b65f48d..f95d588 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -397,7 +397,7 @@ services:
 
   prefect-server:
     container_name: "prefect"
-    image: "prefecthq/prefect:2.19.4-python3.11"
+    image: "prefecthq/prefect:2.19.5-python3.11"
     entrypoint: [ "/opt/prefect/entrypoint.sh", "prefect", "server", "start" ]
     environment:
       - "PREFECT_UI_URL=http://127.0.0.1:4200/api"
@@ -415,7 +415,7 @@ services:
 
   prefect:
     container_name: "prefect-data"
-    image: "prefecthq/prefect:2.19.4-python3.11"
+    image: "prefecthq/prefect:2.19.5-python3.11"
     entrypoint: [ "/opt/prefect/app/start_flows.sh" ]
     environment:
       - "PREFECT_API_URL=http://host.docker.internal:4200/api"

From 904c805685ba8ba93545163b4ad2a0212ef45120 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Fri, 14 Jun 2024 04:46:27 +0000
Subject: [PATCH 18/25] Update apache/druid Docker tag to v30

---
 docker-compose.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index b65f48d..9fe4968 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -612,7 +612,7 @@ services:
 
   druid-coordinator:
     container_name: "druid-coordinator"
-    image: "apache/druid:29.0.1"
+    image: "apache/druid:30.0.0"
     command: [ "coordinator" ]
     volumes:
       - "./data/druid/persist/shared:/opt/shared"
@@ -633,7 +633,7 @@ services:
 
   druid-broker:
     container_name: "druid-broker"
-    image: "apache/druid:29.0.1"
+    image: "apache/druid:30.0.0"
     command: [ "broker" ]
     volumes:
       - "./data/druid/persist/broker_var:/opt/druid/var"
@@ -655,7 +655,7 @@ services:
 
   druid-historical:
     container_name: "druid-historical"
-    image: "apache/druid:29.0.1"
+    image: "apache/druid:30.0.0"
     command: [ "historical" ]
     volumes:
       - "./data/druid/persist/shared:/opt/shared"
@@ -678,7 +678,7 @@ services:
 
   druid-middlemanager:
     container_name: "druid-middlemanager"
-    image: "apache/druid:29.0.1"
+    image: "apache/druid:30.0.0"
     command: [ "middleManager" ]
     volumes:
       - "./data/druid/persist/shared:/opt/shared"
@@ -702,7 +702,7 @@ services:
 
   druid:
     container_name: "druid"
-    image: "apache/druid:29.0.1"
+    image: "apache/druid:30.0.0"
     command: [ "router" ]
     volumes:
       - "./data/druid/persist/router_var:/opt/druid/var"

From 987d8fb5e0d46470d3cd4915a0b52cd8785bb4c9 Mon Sep 17 00:00:00 2001
From: Flook Peter <pflooky@hotmail.com>
Date: Fri, 14 Jun 2024 13:05:54 +0800
Subject: [PATCH 19/25] Add in unitycatalog

---
 README.md           | 1 +
 docker-compose.yaml | 6 ++++++
 run.sh              | 2 ++
 3 files changed, 9 insertions(+)

diff --git a/README.md b/README.md
index b85fd74..71f5e53 100644
--- a/README.md
+++ b/README.md
@@ -91,6 +91,7 @@ If any data is persisted from the services to carry across sessions, it gets pus
 | Database                 | postgres      | ✅         |
 | Database                 | opensearch    | ❌         |
 | Data Catalog             | marquez       | ✅         |
+| Data Catalog             | unitycatalog  | ✅         |
 | Data Catalog             | amundsen      | ❌         |
 | Data Catalog             | datahub       | ❌         |
 | Data Catalog             | openmetadata  | ❌         |
diff --git a/docker-compose.yaml b/docker-compose.yaml
index b65f48d..d187e03 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -247,6 +247,12 @@ services:
       timeout: "5s"
       retries: 3
 
+  unitycatalog:
+    container_name: "unitycatalog"
+    image: "datacatering/unitycatalog:0.1.0"
+    ports:
+      - "8081:8081"
+
   #http
   httpbin:
     container_name: "http"
diff --git a/run.sh b/run.sh
index 71b47ed..fa3ff7c 100755
--- a/run.sh
+++ b/run.sh
@@ -24,6 +24,7 @@ postgres='PGPASSWORD=postgres psql -Upostgres'
 prefect-data='bash'
 presto='presto-cli'
 trino='trino'
+unitycatalog='bash'
 "
 
 usage() {
@@ -32,6 +33,7 @@ usage() {
   echo "    <services>              Name of services to run"
   echo "    -c, connect [service]   Connect to service"
   echo "    -d, down [services...]  Shutdown services (if empty, shutdown all services)"
+  echo "    -h, --help              Show help"
   echo "    -l, list                List supported services"
   echo
   echo "Examples:"

From dacf8996c4fa0221a83528007c14ccaf61370a62 Mon Sep 17 00:00:00 2001
From: Flook Peter <pflooky@hotmail.com>
Date: Fri, 14 Jun 2024 13:50:28 +0800
Subject: [PATCH 20/25] Add docs folder and mkdocs.yaml to top level folder to
 publish docs site

---
 .github/workflows/ci.yaml                     | 29 +++++++++++++++++++
 .../docs => docs}/commands/connect.md         |  0
 .../docs => docs}/commands/list.md            |  0
 .../docs => docs}/commands/shutdown.md        |  0
 .../docs => docs}/commands/start.md           |  0
 .../docs => docs}/customization.md            |  0
 .../docs => docs}/getting-started.md          |  0
 {insta-infra-docs/docs => docs}/index.md      |  0
 {insta-infra-docs/docs => docs}/services.md   |  1 +
 insta-infra-docs/mkdocs.yml => mkdocs.yml     |  1 -
 10 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/ci.yaml
 rename {insta-infra-docs/docs => docs}/commands/connect.md (100%)
 rename {insta-infra-docs/docs => docs}/commands/list.md (100%)
 rename {insta-infra-docs/docs => docs}/commands/shutdown.md (100%)
 rename {insta-infra-docs/docs => docs}/commands/start.md (100%)
 rename {insta-infra-docs/docs => docs}/customization.md (100%)
 rename {insta-infra-docs/docs => docs}/getting-started.md (100%)
 rename {insta-infra-docs/docs => docs}/index.md (100%)
 rename {insta-infra-docs/docs => docs}/services.md (95%)
 rename insta-infra-docs/mkdocs.yml => mkdocs.yml (99%)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 0000000..3ff578e
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,29 @@
+name: ci
+on:
+  push:
+    branches:
+      - master
+      - main
+permissions:
+  contents: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - uses: actions/cache@v4
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - run: pip install mkdocs-material
+      - run: mkdocs gh-deploy --force
\ No newline at end of file
diff --git a/insta-infra-docs/docs/commands/connect.md b/docs/commands/connect.md
similarity index 100%
rename from insta-infra-docs/docs/commands/connect.md
rename to docs/commands/connect.md
diff --git a/insta-infra-docs/docs/commands/list.md b/docs/commands/list.md
similarity index 100%
rename from insta-infra-docs/docs/commands/list.md
rename to docs/commands/list.md
diff --git a/insta-infra-docs/docs/commands/shutdown.md b/docs/commands/shutdown.md
similarity index 100%
rename from insta-infra-docs/docs/commands/shutdown.md
rename to docs/commands/shutdown.md
diff --git a/insta-infra-docs/docs/commands/start.md b/docs/commands/start.md
similarity index 100%
rename from insta-infra-docs/docs/commands/start.md
rename to docs/commands/start.md
diff --git a/insta-infra-docs/docs/customization.md b/docs/customization.md
similarity index 100%
rename from insta-infra-docs/docs/customization.md
rename to docs/customization.md
diff --git a/insta-infra-docs/docs/getting-started.md b/docs/getting-started.md
similarity index 100%
rename from insta-infra-docs/docs/getting-started.md
rename to docs/getting-started.md
diff --git a/insta-infra-docs/docs/index.md b/docs/index.md
similarity index 100%
rename from insta-infra-docs/docs/index.md
rename to docs/index.md
diff --git a/insta-infra-docs/docs/services.md b/docs/services.md
similarity index 95%
rename from insta-infra-docs/docs/services.md
rename to docs/services.md
index 7f18885..81f4b5b 100644
--- a/insta-infra-docs/docs/services.md
+++ b/docs/services.md
@@ -29,4 +29,5 @@
 - rabbitmq
 - solace
 - trino
+- unitycatalog
 - zookeeper
diff --git a/insta-infra-docs/mkdocs.yml b/mkdocs.yml
similarity index 99%
rename from insta-infra-docs/mkdocs.yml
rename to mkdocs.yml
index 0d1edd7..b78d7c8 100644
--- a/insta-infra-docs/mkdocs.yml
+++ b/mkdocs.yml
@@ -11,4 +11,3 @@ nav:
       - List: commands/list.md
   - Customization: customization.md
   - Services: services.md
-

From 479e7d95a74d291f898b6ef7598f9940953c445e Mon Sep 17 00:00:00 2001
From: Flook Peter <pflooky@hotmail.com>
Date: Fri, 14 Jun 2024 13:54:39 +0800
Subject: [PATCH 21/25] Add in toggle for dark mode in docs site

---
 mkdocs.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/mkdocs.yml b/mkdocs.yml
index b78d7c8..eb5fdf4 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -1,6 +1,17 @@
 site_name: insta-infra
 theme:
   name: material
+  palette:
+    # Palette toggle for light mode
+    - scheme: default
+      toggle:
+        icon: material/brightness-7
+        name: Switch to dark mode
+    # Palette toggle for dark mode
+    - scheme: slate
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
 nav:
   - Home: index.md
   - Getting Started: getting-started.md

From 31a45a2e7a223ad0c45a7811d0acd0a195c6e3b4 Mon Sep 17 00:00:00 2001
From: Flook Peter <pflooky@hotmail.com>
Date: Fri, 14 Jun 2024 14:01:39 +0800
Subject: [PATCH 22/25] Use automatic detect prefer color scheme

---
 mkdocs.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/mkdocs.yml b/mkdocs.yml
index eb5fdf4..5add33f 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -2,16 +2,23 @@ site_name: insta-infra
 theme:
   name: material
   palette:
+    # Palette toggle for automatic mode
+    - media: "(prefers-color-scheme)"
+      toggle:
+        icon: material/brightness-auto
+        name: Switch to light mode
     # Palette toggle for light mode
-    - scheme: default
+    - media: "(prefers-color-scheme: light)"
+      scheme: default
       toggle:
         icon: material/brightness-7
         name: Switch to dark mode
     # Palette toggle for dark mode
-    - scheme: slate
+    - media: "(prefers-color-scheme: dark)"
+      scheme: slate
       toggle:
         icon: material/brightness-4
-        name: Switch to light mode
+        name: Switch to system preference
 nav:
   - Home: index.md
   - Getting Started: getting-started.md

From 6e314de39675fa767e7c9f493c49ae6738d24286 Mon Sep 17 00:00:00 2001
From: Flook Peter <pflooky@hotmail.com>
Date: Fri, 14 Jun 2024 14:07:02 +0800
Subject: [PATCH 23/25] Allow for copy of code blocks

---
 mkdocs.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/mkdocs.yml b/mkdocs.yml
index 5add33f..e632960 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -1,6 +1,10 @@
 site_name: insta-infra
 theme:
   name: material
+  features:
+    - content.code.copy
+    - content.code.select
+    - content.tabs.link
   palette:
     # Palette toggle for automatic mode
     - media: "(prefers-color-scheme)"

From e79f0c2acae949683cbf84e009a5ab1c17abf086 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Sat, 15 Jun 2024 21:29:56 +0000
Subject: [PATCH 24/25] Update apache/activemq-artemis Docker tag to v2.35.0

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index a19e94e..75083ca 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -435,7 +435,7 @@ services:
   #messaging
   activemq:
     container_name: "activemq"
-    image: "apache/activemq-artemis:2.34.0"
+    image: "apache/activemq-artemis:2.35.0"
     environment:
       - "ARTEMIS_USER=artemis"
       - "ARTEMIS_PASSWORD=artemis"

From d8373051c43a2f7aba5ea9e50a11c18053ac5d53 Mon Sep 17 00:00:00 2001
From: skhatri <servlet2@msn.com>
Date: Sun, 16 Jun 2024 22:20:57 +1000
Subject: [PATCH 25/25] Add flink taskmanager and jobmanager

---
 README.md           | 75 +++++++++++++++++++++++----------------------
 docker-compose.yaml | 30 +++++++++++++++++-
 run.sh              |  2 ++
 3 files changed, 69 insertions(+), 38 deletions(-)

diff --git a/README.md b/README.md
index 71f5e53..a9ad687 100644
--- a/README.md
+++ b/README.md
@@ -78,40 +78,41 @@ If any data is persisted from the services to carry across sessions, it gets pus
 
 ## Services
 
-| Service Type             | Service       | Supported |
-|--------------------------|---------------|-----------|
-| Change Data Capture      | debezium      | ✅         |
-| Database                 | cassandra     | ✅         |
-| Database                 | cockroachdb   | ✅         |
-| Database                 | elasticsearch | ✅         |
-| Database                 | mariadb       | ✅         |
-| Database                 | mongodb       | ✅         |
-| Database                 | mysql         | ✅         |
-| Database                 | neo4j         | ✅         |
-| Database                 | postgres      | ✅         |
-| Database                 | opensearch    | ❌         |
-| Data Catalog             | marquez       | ✅         |
-| Data Catalog             | unitycatalog  | ✅         |
-| Data Catalog             | amundsen      | ❌         |
-| Data Catalog             | datahub       | ❌         |
-| Data Catalog             | openmetadata  | ❌         |
-| Distributed Coordination | zookeeper     | ✅         |
-| HTTP                     | httpbin       | ✅         |
-| Identity Management      | keycloak      | ✅         |
-| Job Orchestrator         | airflow       | ✅         |
-| Job Orchestrator         | dagster       | ✅         |
-| Job Orchestrator         | mage-ai       | ✅         |
-| Job Orchestrator         | prefect       | ✅         |
-| Messaging                | kafka         | ✅         |
-| Messaging                | solace        | ✅         |
-| Messaging                | activemq      | ✅         |
-| Messaging                | rabbitmq      | ✅         |
-| Object Storage           | minio         | ✅         |
-| Query Engine             | duckdb        | ✅         |
-| Query Engine             | presto        | ✅         |
-| Query Engine             | trino         | ✅         |
-| Real-time OLAP           | clickhouse    | ✅         |
-| Real-time OLAP           | doris         | ✅         |
-| Real-time OLAP           | druid         | ✅         |
-| Real-time OLAP           | pinot         | ✅         |
-| Test Data Management     | data-caterer  | ✅         |
+| Service Type               | Service       | Supported |
+|----------------------------|---------------|-----------|
+| Change Data Capture        | debezium      | ✅         |
+| Database                   | cassandra     | ✅         |
+| Database                   | cockroachdb   | ✅         |
+| Database                   | elasticsearch | ✅         |
+| Database                   | mariadb       | ✅         |
+| Database                   | mongodb       | ✅         |
+| Database                   | mysql         | ✅         |
+| Database                   | neo4j         | ✅         |
+| Database                   | postgres      | ✅         |
+| Database                   | opensearch    | ❌         |
+| Data Catalog               | marquez       | ✅         |
+| Data Catalog               | unitycatalog  | ✅         |
+| Data Catalog               | amundsen      | ❌         |
+| Data Catalog               | datahub       | ❌         |
+| Data Catalog               | openmetadata  | ❌         |
+| Distributed Coordination   | zookeeper     | ✅         |
+| Distribute Data Processing | flink         | ✅         |
+| HTTP                       | httpbin       | ✅         |
+| Identity Management        | keycloak      | ✅         |
+| Job Orchestrator           | airflow       | ✅         |
+| Job Orchestrator           | dagster       | ✅         |
+| Job Orchestrator           | mage-ai       | ✅         |
+| Job Orchestrator           | prefect       | ✅         |
+| Messaging                  | kafka         | ✅         |
+| Messaging                  | solace        | ✅         |
+| Messaging                  | activemq      | ✅         |
+| Messaging                  | rabbitmq      | ✅         |
+| Object Storage             | minio         | ✅         |
+| Query Engine               | duckdb        | ✅         |
+| Query Engine               | presto        | ✅         |
+| Query Engine               | trino         | ✅         |
+| Real-time OLAP             | clickhouse    | ✅         |
+| Real-time OLAP             | doris         | ✅         |
+| Real-time OLAP             | druid         | ✅         |
+| Real-time OLAP             | pinot         | ✅         |
+| Test Data Management       | data-caterer  | ✅         |
diff --git a/docker-compose.yaml b/docker-compose.yaml
index a19e94e..e9ce16d 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -802,4 +802,32 @@ services:
     depends_on:
       postgres:
         condition: "service_healthy"
-  
+
+  #flink containers
+  flink-jobmanager:
+    image: flink:1.19.0-scala_2.12-java17
+    container_name: flink-jobmanager
+    expose:
+      - "6123"
+    ports:
+      - "8081:8081"
+    command: jobmanager
+    environment:
+      - JOB_MANAGER_RPC_ADDRESS=flink-jobmanager
+      - FLINK_PROPERTIES=jobmanager.rpc.address:flink-jobmanager
+
+  flink:
+    image: flink:1.19.0-scala_2.12-java17
+    container_name: flink
+    expose:
+      - "6121"
+      - "6122"
+    depends_on:
+      - flink-jobmanager
+    command: taskmanager
+    links: []
+    environment:
+      - JOB_MANAGER_RPC_ADDRESS=flink-jobmanager
+      - FLINK_PROPERTIES=jobmanager.rpc.address:flink-jobmanager
+
+
diff --git a/run.sh b/run.sh
index fa3ff7c..c4ffff5 100755
--- a/run.sh
+++ b/run.sh
@@ -25,6 +25,8 @@ prefect-data='bash'
 presto='presto-cli'
 trino='trino'
 unitycatalog='bash'
+flink='bash'
+flink-jobmanager='bash'
 "
 
 usage() {