Merge pull request #54 from data-catering/maestro

Add in maestro
data-catering · Jul 23, 2024 · 96572d7 · 96572d7
2 parents 22825d3 + 0b34e01
commit 96572d7
Show file tree

Hide file tree

Showing 5 changed files with 83 additions and 45 deletions.
diff --git a/README.md b/README.md
@@ -113,48 +113,49 @@ POSTGRES_USER=my-user POSTGRES_PASSWORD=my-password ./run.sh postgres
 
 ## Services
 
-| Service Type                | Service            | Supported |
-|-----------------------------|--------------------|-----------|
-| Change Data Capture         | debezium           | ✅         |
-| Database                    | cassandra          | ✅         |
-| Database                    | cockroachdb        | ✅         |
-| Database                    | elasticsearch      | ✅         |
-| Database                    | mariadb            | ✅         |
-| Database                    | mongodb            | ✅         |
-| Database                    | mssql              | ✅         |
-| Database                    | mysql              | ✅         |
-| Database                    | neo4j              | ✅         |
-| Database                    | postgres           | ✅         |
-| Database                    | spanner            | ✅         |
-| Database                    | sqlite             | ✅         |
-| Database                    | opensearch         | ❌         |
-| Data Catalog                | marquez            | ✅         |
-| Data Catalog                | unitycatalog       | ✅         |
-| Data Catalog                | amundsen           | ❌         |
-| Data Catalog                | datahub            | ❌         |
-| Data Catalog                | openmetadata       | ❌         |
-| Distributed Coordination    | zookeeper          | ✅         |
-| Distributed Data Processing | flink              | ✅         |
-| HTTP                        | httpbin            | ✅         |
-| Identity Management         | keycloak           | ✅         |
-| Job Orchestrator            | airflow            | ✅         |
-| Job Orchestrator            | dagster            | ✅         |
-| Job Orchestrator            | mage-ai            | ✅         |
-| Job Orchestrator            | prefect            | ✅         |
-| Messaging                   | activemq           | ✅         |
-| Messaging                   | kafka              | ✅         |
-| Messaging                   | rabbitmq           | ✅         |
-| Messaging                   | solace             | ✅         |
-| Notebook                    | jupyter            | ✅         |
-| Object Storage              | minio              | ✅         |
-| Query Engine                | duckdb             | ✅         |
-| Query Engine                | flight-sql         | ✅         |
-| Query Engine                | presto             | ✅         |
-| Query Engine                | trino              | ✅         |
-| Real-time OLAP              | clickhouse         | ✅         |
-| Real-time OLAP              | doris              | ✅         |
-| Real-time OLAP              | druid              | ✅         |
-| Real-time OLAP              | pinot              | ✅         |
-| Test Data Management        | data-caterer       | ✅         |
-| Workflow                    | temporal           | ✅         | 
+| Service Type                | Service       | Supported |
+|-----------------------------|---------------|-----------|
+| Change Data Capture         | debezium      | ✅         |
+| Database                    | cassandra     | ✅         |
+| Database                    | cockroachdb   | ✅         |
+| Database                    | elasticsearch | ✅         |
+| Database                    | mariadb       | ✅         |
+| Database                    | mongodb       | ✅         |
+| Database                    | mssql         | ✅         |
+| Database                    | mysql         | ✅         |
+| Database                    | neo4j         | ✅         |
+| Database                    | postgres      | ✅         |
+| Database                    | spanner       | ✅         |
+| Database                    | sqlite        | ✅         |
+| Database                    | opensearch    | ❌         |
+| Data Catalog                | marquez       | ✅         |
+| Data Catalog                | unitycatalog  | ✅         |
+| Data Catalog                | amundsen      | ❌         |
+| Data Catalog                | datahub       | ❌         |
+| Data Catalog                | openmetadata  | ❌         |
+| Distributed Coordination    | zookeeper     | ✅         |
+| Distributed Data Processing | flink         | ✅         |
+| HTTP                        | httpbin       | ✅         |
+| Identity Management         | keycloak      | ✅         |
+| Job Orchestrator            | airflow       | ✅         |
+| Job Orchestrator            | dagster       | ✅         |
+| Job Orchestrator            | mage-ai       | ✅         |
+| Job Orchestrator            | prefect       | ✅         |
+| Messaging                   | activemq      | ✅         |
+| Messaging                   | kafka         | ✅         |
+| Messaging                   | rabbitmq      | ✅         |
+| Messaging                   | solace        | ✅         |
+| Notebook                    | jupyter       | ✅         |
+| Object Storage              | minio         | ✅         |
+| Query Engine                | duckdb        | ✅         |
+| Query Engine                | flight-sql    | ✅         |
+| Query Engine                | presto        | ✅         |
+| Query Engine                | trino         | ✅         |
+| Real-time OLAP              | clickhouse    | ✅         |
+| Real-time OLAP              | doris         | ✅         |
+| Real-time OLAP              | druid         | ✅         |
+| Real-time OLAP              | pinot         | ✅         |
+| Test Data Management        | data-caterer  | ✅         |
+| Workflow                    | maestro       | ✅         | 
+| Workflow                    | temporal      | ✅         | 
 
diff --git a/data/cockroachdb/data/my_data.sql b/data/cockroachdb/data/my_data.sql
@@ -0,0 +1 @@
+CREATE DATABASE maestro;
diff --git a/data/cockroachdb/init.sh b/data/cockroachdb/init.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+count=0
+total=0
+
+for f in $(ls /tmp/data/*.sql);
+do
+  total=$((total + 1))
+  cockroach sql --insecure --host cockroachdb < "${f}"
+  if [[ $? -eq 0 ]];
+  then
+    count=$((count + 1))
+  fi;
+done;
+
+echo "Executed ${count} out of ${total} files"
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -154,6 +154,16 @@ services:
       - "9000:9000"
     user: "101:101"
   cockroachdb:
+    command: [bash, -c, /tmp/scripts/init.sh]
+    container_name: cockroachdb-data
+    depends_on:
+      cockroachdb-server:
+        condition: service_healthy
+    image: "cockroachdb/cockroach:${COCKROACHDB_VERSION:-v24.1.0}"
+    volumes:
+      - "./data/cockroachdb/init.sh:/tmp/scripts/init.sh"
+      - "${COCKROACHDB_DATA:-./data/cockroachdb/data}:/tmp/data"
+  cockroachdb-server:
     command: [start-single-node, --insecure]
     container_name: cockroachdb
     healthcheck:
@@ -478,6 +488,17 @@ services:
     restart: unless-stopped
     volumes:
       - "./data/keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro"
+  maestro:
+    container_name: maestro
+    depends_on:
+      cockroachdb:
+        condition: service_completed_successfully
+    environment:
+      - CONDUCTOR_CONFIGS_JDBCURL=jdbc:postgresql://cockroachdb:26257/maestro
+      - CONDUCTOR_CONFIGS_JDBCUSERNAME=root
+    image: "datacatering/maestro:${MAESTRO_VERSION:-0.1.0}"
+    ports:
+      - "8081:8080"
   mage-ai:
     command: mage start your_first_project
     container_name: mage-ai

diff --git a/run.sh b/run.sh
@@ -115,7 +115,6 @@ check_persist_flag() {
 
 startup_services() {
   echo -e "${GREEN}Starting up services...${NC}"
-  echo "$COMPOSE_FILES"
   docker-compose $COMPOSE_FILES up -d --quiet-pull "$@"
   if [ $? != 0 ]; then
     echo -e "${RED}Error: Failed to start up services${NC}"