docker-compose.yml

# https://github.com/jgoodman8/zeppelin-spark-standalone-cluster
# Before launching the containers
# docker run --name=master -d bde2020/spark-master:2.3.0-hadoop2.7
# docker cp master:/spark spark
# docker stop master
# Chown algoproto.algoproto notebooks -R
# Chmod 777 notebooks -R
---
version: '2'

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:5.4.1
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  broker:
    image: confluentinc/cp-enterprise-kafka:5.4.1
    hostname: broker
    container_name: broker
    depends_on:
      - zookeeper
    ports:
      - "29092:29092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1

  schema-registry:
    image: confluentinc/cp-schema-registry:5.4.1
    hostname: schema-registry
    container_name: schema-registry
    depends_on:
      - zookeeper
      - broker
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
   
  rstudio:
    image: hikagenji/rstudio:latest
    hostname: rstudio
    container_name: rstudio
    ports:
      - "8787:8787"
    environment:
      PASSWORD: pass
      ROOT: "TRUE"
      
  spark-master:
    image: hikagenji/spark-master:latest
    container_name: spark-master
    expose:
      - "7077"
    ports:
      - "8181:8080"
      - "7077:7077"
    environment:
      - INIT_DAEMON_STEP=setup_spark
    volumes:
      - ./spark:/spark
      
  spark-worker:
    image: hikagenji/spark-worker:latest
    container_name: spark-worker
    depends_on:
      - spark-master
    ports:
      - "8082:8082"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
      
  zeppelin:
    image: apache/zeppelin:0.9.0
    container_name: zeppelin
    ports:
      - "8080:8080"
    volumes:
      - ./spark:/zeppelin/spark
      - ./notebooks:/zeppelin/notebook
    environment:
      - "MASTER=spark://spark-master:7077"
      - "SPARK_MASTER=spark://spark-master:7077"
      - "SPARK_HOME=/zeppelin/spark"
      - "SPARK_SUBMIT_OPTIONS=--packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.5,org.apache.spark:spark-avro_2.11:2.4.5"
      
  redis:
    image: 'docker.io/bitnami/redis:6.0-debian-10'
    environment:
      # ALLOW_EMPTY_PASSWORD is recommended only for development.
      - ALLOW_EMPTY_PASSWORD=yes
      - REDIS_DISABLE_COMMANDS=FLUSHDB,FLUSHALL
    ports:
      - '6379:6379'
    volumes:
      - 'redis_data:/bitnami/redis/data'
  redis-commander:
    image: rediscommander/redis-commander:latest
    depends_on:
      - redis
    environment:
      - REDIS_HOSTS=redis:redis
    ports:
      - 6380:8081
volumes:
  redis_data:
    driver: local