Merge pull request #40 from skhatri/feature/add-flink

Add flink taskmanager and jobmanager
data-catering · Jun 17, 2024 · 23345f9 · 23345f9
2 parents 6e314de + d837305
commit 23345f9
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 38 deletions.
diff --git a/README.md b/README.md
@@ -78,40 +78,41 @@ If any data is persisted from the services to carry across sessions, it gets pus
 
 ## Services
 
-| Service Type             | Service       | Supported |
-|--------------------------|---------------|-----------|
-| Change Data Capture      | debezium      | ✅         |
-| Database                 | cassandra     | ✅         |
-| Database                 | cockroachdb   | ✅         |
-| Database                 | elasticsearch | ✅         |
-| Database                 | mariadb       | ✅         |
-| Database                 | mongodb       | ✅         |
-| Database                 | mysql         | ✅         |
-| Database                 | neo4j         | ✅         |
-| Database                 | postgres      | ✅         |
-| Database                 | opensearch    | ❌         |
-| Data Catalog             | marquez       | ✅         |
-| Data Catalog             | unitycatalog  | ✅         |
-| Data Catalog             | amundsen      | ❌         |
-| Data Catalog             | datahub       | ❌         |
-| Data Catalog             | openmetadata  | ❌         |
-| Distributed Coordination | zookeeper     | ✅         |
-| HTTP                     | httpbin       | ✅         |
-| Identity Management      | keycloak      | ✅         |
-| Job Orchestrator         | airflow       | ✅         |
-| Job Orchestrator         | dagster       | ✅         |
-| Job Orchestrator         | mage-ai       | ✅         |
-| Job Orchestrator         | prefect       | ✅         |
-| Messaging                | kafka         | ✅         |
-| Messaging                | solace        | ✅         |
-| Messaging                | activemq      | ✅         |
-| Messaging                | rabbitmq      | ✅         |
-| Object Storage           | minio         | ✅         |
-| Query Engine             | duckdb        | ✅         |
-| Query Engine             | presto        | ✅         |
-| Query Engine             | trino         | ✅         |
-| Real-time OLAP           | clickhouse    | ✅         |
-| Real-time OLAP           | doris         | ✅         |
-| Real-time OLAP           | druid         | ✅         |
-| Real-time OLAP           | pinot         | ✅         |
-| Test Data Management     | data-caterer  | ✅         |
+| Service Type               | Service       | Supported |
+|----------------------------|---------------|-----------|
+| Change Data Capture        | debezium      | ✅         |
+| Database                   | cassandra     | ✅         |
+| Database                   | cockroachdb   | ✅         |
+| Database                   | elasticsearch | ✅         |
+| Database                   | mariadb       | ✅         |
+| Database                   | mongodb       | ✅         |
+| Database                   | mysql         | ✅         |
+| Database                   | neo4j         | ✅         |
+| Database                   | postgres      | ✅         |
+| Database                   | opensearch    | ❌         |
+| Data Catalog               | marquez       | ✅         |
+| Data Catalog               | unitycatalog  | ✅         |
+| Data Catalog               | amundsen      | ❌         |
+| Data Catalog               | datahub       | ❌         |
+| Data Catalog               | openmetadata  | ❌         |
+| Distributed Coordination   | zookeeper     | ✅         |
+| Distribute Data Processing | flink         | ✅         |
+| HTTP                       | httpbin       | ✅         |
+| Identity Management        | keycloak      | ✅         |
+| Job Orchestrator           | airflow       | ✅         |
+| Job Orchestrator           | dagster       | ✅         |
+| Job Orchestrator           | mage-ai       | ✅         |
+| Job Orchestrator           | prefect       | ✅         |
+| Messaging                  | kafka         | ✅         |
+| Messaging                  | solace        | ✅         |
+| Messaging                  | activemq      | ✅         |
+| Messaging                  | rabbitmq      | ✅         |
+| Object Storage             | minio         | ✅         |
+| Query Engine               | duckdb        | ✅         |
+| Query Engine               | presto        | ✅         |
+| Query Engine               | trino         | ✅         |
+| Real-time OLAP             | clickhouse    | ✅         |
+| Real-time OLAP             | doris         | ✅         |
+| Real-time OLAP             | druid         | ✅         |
+| Real-time OLAP             | pinot         | ✅         |
+| Test Data Management       | data-caterer  | ✅         |
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -802,4 +802,32 @@ services:
     depends_on:
       postgres:
         condition: "service_healthy"
-
+
+  #flink containers
+  flink-jobmanager:
+    image: flink:1.19.0-scala_2.12-java17
+    container_name: flink-jobmanager
+    expose:
+      - "6123"
+    ports:
+      - "8081:8081"
+    command: jobmanager
+    environment:
+      - JOB_MANAGER_RPC_ADDRESS=flink-jobmanager
+      - FLINK_PROPERTIES=jobmanager.rpc.address:flink-jobmanager
+
+  flink:
+    image: flink:1.19.0-scala_2.12-java17
+    container_name: flink
+    expose:
+      - "6121"
+      - "6122"
+    depends_on:
+      - flink-jobmanager
+    command: taskmanager
+    links: []
+    environment:
+      - JOB_MANAGER_RPC_ADDRESS=flink-jobmanager
+      - FLINK_PROPERTIES=jobmanager.rpc.address:flink-jobmanager
+
+
diff --git a/run.sh b/run.sh
@@ -25,6 +25,8 @@ prefect-data='bash'
 presto='presto-cli'
 trino='trino'
 unitycatalog='bash'
+flink='bash'
+flink-jobmanager='bash'
 "
 
 usage() {