docker-compose.yml

version: "3.7"

services:
  # This service runs the postgres DB used by dagster for run storage, schedule storage,
  # and event log storage.
  dagster_postgresql:
    image: postgres:11
    container_name: dagster_postgresql
    environment:
      POSTGRES_USER: ${DAGSTER_POSTGRES_USER}
      POSTGRES_PASSWORD: ${DAGSTER_POSTGRES_PASSWORD}
      POSTGRES_DB: ${DAGSTER_POSTGRES_DB}
    volumes:
      - pg_dagster:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    networks:
      - aod_downloader_network

  # This service runs the postgres DB for collect data from albion-data-project
  dwh_postgresql:
    image: postgres:11
    container_name: dwh_postgresql
    environment:
      POSTGRES_USER: ${DWH_POSTGRES_USER}
      POSTGRES_PASSWORD: ${DWH_POSTGRES_PASSWORD}
      POSTGRES_DB: ${DWH_POSTGRES_DB}
    volumes:
      - pg_dwh:/var/lib/postgresql/data
    ports:
      - "5433:5432" # Use a different port to avoid conflicts
    networks:
      - aod_downloader_network

  # This service runs the gRPC server that loads your user code, in both dagster-webserver
  # and dagster-daemon. By setting DAGSTER_CURRENT_IMAGE to its own image, we tell the
  # run launcher to use this same image when launching runs in a new container as well.
  # Multiple containers like this can be deployed separately - each just needs to run on
  # its own port, and have its own entry in the workspace.yaml file that's loaded by the
      # webserver.
  dagster_user_code:
    build:
      context: .
      dockerfile: ./Dockerfile_user_code
    container_name: dagster_user_code
    image: dagster_user_code_image
    restart: always
    environment:
      DAGSTER_POSTGRES_USER: ${DAGSTER_POSTGRES_USER}
      DAGSTER_POSTGRES_PASSWORD: ${DAGSTER_POSTGRES_PASSWORD}
      DAGSTER_POSTGRES_DB: ${DAGSTER_POSTGRES_DB}
      DAGSTER_CURRENT_IMAGE: "dagster_user_code_image"
      DWH_POSTGRES_HOST: ${DWH_POSTGRES_HOST}
      DWH_POSTGRES_USER: ${DWH_POSTGRES_USER}
      DWH_POSTGRES_PASSWORD: ${DWH_POSTGRES_PASSWORD}
      DWH_POSTGRES_DB: ${DWH_POSTGRES_DB}
      DWH_POSTGRES_PORT: ${DWH_POSTGRES_PORT}
    volumes:
      - ./app:/opt/dagster/app
    networks:
      - aod_downloader_network

  # This service runs dagster-webserver, which loads your user code from the user code container.
  # Since our instance uses the QueuedRunCoordinator, any runs submitted from the webserver will be put on
  # a queue and later dequeued and launched by dagster-daemon.
  dagster_webserver:
    build:
      context: .
      dockerfile: ./Dockerfile_dagster
    entrypoint:
      - dagster-webserver
      - -h
      - "0.0.0.0"
      - -p
      - "3000"
      - -w
      - workspace.yaml
    container_name: dagster_webserver
    expose:
      - "3000"
    ports:
      - "3000:3000"
    environment:
      DAGSTER_POSTGRES_USER: ${DAGSTER_POSTGRES_USER}
      DAGSTER_POSTGRES_PASSWORD: ${DAGSTER_POSTGRES_PASSWORD}
      DAGSTER_POSTGRES_DB: ${DAGSTER_POSTGRES_DB}
    volumes: # Make docker client accessible so we can terminate containers from the webserver
      - /var/run/docker.sock:/var/run/docker.sock
      - /tmp/io_manager_storage:/tmp/io_manager_storage
    networks:
      - aod_downloader_network
    depends_on:
      - dagster_postgresql
      - dagster_user_code

  # This service runs the dagster-daemon process, which is responsible for taking runs
  # off of the queue and launching them, as well as creating runs from schedules or sensors.
  dagster_daemon:
    build:
      context: .
      dockerfile: ./Dockerfile_dagster
    entrypoint:
      - dagster-daemon
      - run
    container_name: dagster_daemon
    restart: on-failure
    environment:
      DAGSTER_POSTGRES_USER: ${DAGSTER_POSTGRES_USER}
      DAGSTER_POSTGRES_PASSWORD: ${DAGSTER_POSTGRES_PASSWORD}
      DAGSTER_POSTGRES_DB: ${DAGSTER_POSTGRES_DB}
    volumes: # Make docker client accessible so we can launch containers using host docker
      - /var/run/docker.sock:/var/run/docker.sock
      - /tmp/io_manager_storage:/tmp/io_manager_storage
    networks:
      - aod_downloader_network
    depends_on:
      - dagster_postgresql
      - dagster_user_code

networks:
  aod_downloader_network:
    driver: bridge
    name: aod_downloader_network

volumes:
  pg_dwh:
  pg_dagster: