From d8373051c43a2f7aba5ea9e50a11c18053ac5d53 Mon Sep 17 00:00:00 2001 From: skhatri Date: Sun, 16 Jun 2024 22:20:57 +1000 Subject: [PATCH] Add flink taskmanager and jobmanager --- README.md | 75 +++++++++++++++++++++++---------------------- docker-compose.yaml | 30 +++++++++++++++++- run.sh | 2 ++ 3 files changed, 69 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 71f5e53..a9ad687 100644 --- a/README.md +++ b/README.md @@ -78,40 +78,41 @@ If any data is persisted from the services to carry across sessions, it gets pus ## Services -| Service Type | Service | Supported | -|--------------------------|---------------|-----------| -| Change Data Capture | debezium | ✅ | -| Database | cassandra | ✅ | -| Database | cockroachdb | ✅ | -| Database | elasticsearch | ✅ | -| Database | mariadb | ✅ | -| Database | mongodb | ✅ | -| Database | mysql | ✅ | -| Database | neo4j | ✅ | -| Database | postgres | ✅ | -| Database | opensearch | ❌ | -| Data Catalog | marquez | ✅ | -| Data Catalog | unitycatalog | ✅ | -| Data Catalog | amundsen | ❌ | -| Data Catalog | datahub | ❌ | -| Data Catalog | openmetadata | ❌ | -| Distributed Coordination | zookeeper | ✅ | -| HTTP | httpbin | ✅ | -| Identity Management | keycloak | ✅ | -| Job Orchestrator | airflow | ✅ | -| Job Orchestrator | dagster | ✅ | -| Job Orchestrator | mage-ai | ✅ | -| Job Orchestrator | prefect | ✅ | -| Messaging | kafka | ✅ | -| Messaging | solace | ✅ | -| Messaging | activemq | ✅ | -| Messaging | rabbitmq | ✅ | -| Object Storage | minio | ✅ | -| Query Engine | duckdb | ✅ | -| Query Engine | presto | ✅ | -| Query Engine | trino | ✅ | -| Real-time OLAP | clickhouse | ✅ | -| Real-time OLAP | doris | ✅ | -| Real-time OLAP | druid | ✅ | -| Real-time OLAP | pinot | ✅ | -| Test Data Management | data-caterer | ✅ | +| Service Type | Service | Supported | +|----------------------------|---------------|-----------| +| Change Data Capture | debezium | ✅ | +| Database | cassandra | ✅ | +| Database | cockroachdb | ✅ | +| Database | elasticsearch | ✅ | +| Database | mariadb | ✅ | +| Database | mongodb | ✅ | +| Database | mysql | ✅ | +| Database | neo4j | ✅ | +| Database | postgres | ✅ | +| Database | opensearch | ❌ | +| Data Catalog | marquez | ✅ | +| Data Catalog | unitycatalog | ✅ | +| Data Catalog | amundsen | ❌ | +| Data Catalog | datahub | ❌ | +| Data Catalog | openmetadata | ❌ | +| Distributed Coordination | zookeeper | ✅ | +| Distribute Data Processing | flink | ✅ | +| HTTP | httpbin | ✅ | +| Identity Management | keycloak | ✅ | +| Job Orchestrator | airflow | ✅ | +| Job Orchestrator | dagster | ✅ | +| Job Orchestrator | mage-ai | ✅ | +| Job Orchestrator | prefect | ✅ | +| Messaging | kafka | ✅ | +| Messaging | solace | ✅ | +| Messaging | activemq | ✅ | +| Messaging | rabbitmq | ✅ | +| Object Storage | minio | ✅ | +| Query Engine | duckdb | ✅ | +| Query Engine | presto | ✅ | +| Query Engine | trino | ✅ | +| Real-time OLAP | clickhouse | ✅ | +| Real-time OLAP | doris | ✅ | +| Real-time OLAP | druid | ✅ | +| Real-time OLAP | pinot | ✅ | +| Test Data Management | data-caterer | ✅ | diff --git a/docker-compose.yaml b/docker-compose.yaml index a19e94e..e9ce16d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -802,4 +802,32 @@ services: depends_on: postgres: condition: "service_healthy" - + + #flink containers + flink-jobmanager: + image: flink:1.19.0-scala_2.12-java17 + container_name: flink-jobmanager + expose: + - "6123" + ports: + - "8081:8081" + command: jobmanager + environment: + - JOB_MANAGER_RPC_ADDRESS=flink-jobmanager + - FLINK_PROPERTIES=jobmanager.rpc.address:flink-jobmanager + + flink: + image: flink:1.19.0-scala_2.12-java17 + container_name: flink + expose: + - "6121" + - "6122" + depends_on: + - flink-jobmanager + command: taskmanager + links: [] + environment: + - JOB_MANAGER_RPC_ADDRESS=flink-jobmanager + - FLINK_PROPERTIES=jobmanager.rpc.address:flink-jobmanager + + diff --git a/run.sh b/run.sh index fa3ff7c..c4ffff5 100755 --- a/run.sh +++ b/run.sh @@ -25,6 +25,8 @@ prefect-data='bash' presto='presto-cli' trino='trino' unitycatalog='bash' +flink='bash' +flink-jobmanager='bash' " usage() {