docker-compose.yml

version: '3'

volumes:
  # It sits by default in /var/lib/docker/volumes/
  shared-workspace:
    driver: local
    driver_opts:
      type: none
      # We need this since $PWD does not work when running Docker with sudo
      device: ${LOCAL_WORKSPACE}
      o: bind

services:
  spark-master:
    image: spark-local-spark-master
    container_name: spark-master
    env_file:
      - ./spark.env
    ports:
      - 8080:8080
      - 7077:7077
    volumes:
      - shared-workspace:${SHARED_WORKSPACE}
  spark-worker-1:
    image: spark-local-spark-worker
    container_name: spark-worker-1
    env_file:
      - ./spark.env
    depends_on:
      - spark-master
    ports:
      - 8081:8081
    volumes:
      - shared-workspace:${SHARED_WORKSPACE}
  spark-worker-2:
    image: spark-local-spark-worker
    container_name: spark-worker-2
    env_file:
      - ./spark.env
    depends_on:
      - spark-master
    ports:
      - 8082:8081
    volumes:
      - shared-workspace:${SHARED_WORKSPACE}
  jupyterlab:
    image: spark-local-jupyterlab
    container_name: spark-jupyterlab
    depends_on:
      - spark-master
    ports:
      - 8888:8888
    volumes:
      - shared-workspace:${SHARED_WORKSPACE}