docker-compose.yml

version: '3'

services:
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java11
    container_name: namenode
    ports:
      - "9870:9870"
    environment:
      - CLUSTER_NAME=hadoop-cluster
    volumes:
      - ./hadoop-namenode-data:/hadoop/dfs/name

  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java11
    container_name: datanode
    depends_on:
      - namenode
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:9000
    volumes:
      - ./hadoop-datanode-data:/hadoop/dfs/data

  spark-master:
    image: bitnami/spark:3.3.0-debian-10-r0
    container_name: spark-master
    depends_on:
      - namenode
      - datanode
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_CLEANER_ENABLED=no
      - SPARK_HISTORY_FS_LOGDIRECTORY=hdfs://namenode:9000/spark-history
    ports:
      - "4040:4040"
      - "7077:7077"
      - "8080:8080"

  spark-worker:
    image: bitnami/spark:3.3.0-debian-10-r0
    container_name: spark-worker
    depends_on:
      - spark-master
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_CLEANER_ENABLED=no
    volumes:
      - ./spark-apps:/opt/bitnami/spark/spark-apps

  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
    ports:
      - "2181:2181"

  kafka:
    image: confluentinc/cp-kafka:latest
    container_name: kafka
    depends_on:
      - zookeeper
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    ports:
      - "9092:9092"