Merge pull request #1490 from codalab/develop

Merge develop into master (/!\ Poetry /!\)

Caddyfile

@@ -29,7 +29,8 @@
 
   # Logs:
   log {
-    output stdout
-    format console
+    output file /var/log/caddyaccess.log {
+      roll_disabled
+    }
   }
 }

Dockerfile

@@ -1,12 +1,20 @@
-FROM python:3.8
+FROM python:3.9
 
 RUN apt-get update && apt-get install -y gcc build-essential && rm -rf /var/lib/apt/lists/*
 
 ENV PYTHONUNBUFFERED 1
 
-ADD requirements.dev.txt .
-ADD requirements.txt .
-RUN pip install -U pip
-RUN pip install -r requirements.dev.txt
+
+RUN curl -sSL https://install.python-poetry.org | python3 -
+# Poetry location so future commands (below) work
+ENV PATH $PATH:/root/.local/bin
+# Want poetry to use system python of docker container
+RUN poetry config virtualenvs.create false
+RUN poetry config virtualenvs.in-project false
+
+COPY pyproject.toml ./
+COPY poetry.lock ./
+
+RUN poetry install
 
 WORKDIR /app

Dockerfile.compute_worker

@@ -1,13 +1,21 @@
-FROM --platform=linux/amd64 python:3.8
+FROM --platform=linux/amd64 python:3.9
 
 # This makes output not buffer and return immediately, nice for seeing results in stdout
 ENV PYTHONUNBUFFERED 1
 
 # Install Docker
 RUN apt-get update && curl -fsSL https://get.docker.com | sh
 
-ADD compute_worker/compute_worker_requirements.txt .
-RUN pip install -r compute_worker_requirements.txt
+
+RUN curl -sSL https://install.python-poetry.org | python3 -
+# Poetry location so future commands (below) work
+ENV PATH $PATH:/root/.local/bin
+# Want poetry to use system python of docker container
+RUN poetry config virtualenvs.create false
+RUN poetry config virtualenvs.in-project false
+COPY ./compute_worker/pyproject.toml ./
+COPY ./compute_worker/poetry.lock ./
+RUN poetry install
 
 ADD compute_worker .
 

Dockerfile.compute_worker_gpu

@@ -1,26 +1,34 @@
-FROM --platform=linux/amd64 python:3.8.1-buster
-
-# We need curl to get docker/nvidia-docker
-RUN apt-get update &&  apt-get install curl wget -y
+FROM --platform=linux/amd64 python:3.9
 
 # This makes output not buffer and return immediately, nice for seeing results in stdout
 ENV PYTHONUNBUFFERED 1
 
 # Install Docker
 RUN apt-get update && curl -fsSL https://get.docker.com | sh
 
-# nvidia-docker jazz
-RUN curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add -
-RUN curl -s -L https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | \
-  tee /etc/apt/sources.list.d/nvidia-docker.list
-RUN apt-get update && apt-get install -y nvidia-docker2
 
-# make it explicit that we're using GPUs
-ENV NVIDIA_DOCKER 1
+# Nvidia Container Toolkit for cuda use with docker
+# [source](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
+RUN curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
+    && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
+    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
+    tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
+RUN apt-get update -y;
+RUN apt-get install -y nvidia-container-toolkit
+# Make it explicit that we're using GPUs
+# BB - not convinced we need this
+ENV USE_GPU 1
+
+RUN curl -sSL https://install.python-poetry.org | python3 -
+# Poetry location so future commands (below) work
+ENV PATH $PATH:/root/.local/bin
+# Want poetry to use system python of docker container
+RUN poetry config virtualenvs.create false
+RUN poetry config virtualenvs.in-project false
+COPY ./compute_worker/pyproject.toml ./
+COPY ./compute_worker/poetry.lock ./
+RUN poetry install
 
-# Python reqs and actual worker stuff
-ADD compute_worker/compute_worker_requirements.txt .
-RUN pip3 install -r compute_worker_requirements.txt
 ADD compute_worker .
 
 CMD celery -A compute_worker worker \

Dockerfile.flower

@@ -1,14 +1,28 @@
 FROM python:3.9-alpine
 
 # Get latest root certificates
-RUN apk add --no-cache ca-certificates && update-ca-certificates
+RUN apk add --no-cache ca-certificates && update-ca-certificates curl
+RUN apk add curl
 
-# Install the required packages
-RUN pip install --no-cache-dir redis==3.0.1 flower==0.9.3 "celery<5.0.0"
+# # Install the required packages
+RUN curl -sSL https://install.python-poetry.org | python3 -
+# Poetry location so future commands (below) work
+ENV PATH $PATH:/root/.local/bin
+# Want poetry to use system python of docker container
+RUN poetry config virtualenvs.create false
+RUN poetry config virtualenvs.in-project false
+
+RUN poetry init --no-interaction
+
+RUN poetry add redis=3.0.1
+RUN poetry add flower=0.9.3
+RUN poetry add celery="<5.0.0"
 
 # PYTHONUNBUFFERED: Force stdin, stdout and stderr to be totally unbuffered. (equivalent to `python -u`)
 # PYTHONHASHSEED: Enable hash randomization (equivalent to `python -R`)
 # PYTHONDONTWRITEBYTECODE: Do not write byte files to disk, since we maintain it as readonly. (equivalent to `python -B`)
+
+
 ENV PYTHONUNBUFFERED=1 PYTHONHASHSEED=random PYTHONDONTWRITEBYTECODE=1
 
 # Default port

compute_worker/compute_worker.py

@@ -82,9 +82,6 @@
 # Setup the container engine that we are using
 if os.environ.get("CONTAINER_ENGINE_EXECUTABLE"):
     CONTAINER_ENGINE_EXECUTABLE = os.environ.get("CONTAINER_ENGINE_EXECUTABLE")
-# We could probably depreciate this now that we can specify the executable
-elif os.environ.get("NVIDIA_DOCKER"):
-    CONTAINER_ENGINE_EXECUTABLE = "nvidia-docker"
 else:
     CONTAINER_ENGINE_EXECUTABLE = "docker"
 
@@ -654,6 +651,10 @@ async def _run_program_directory(self, program_dir, kind, can_be_output=False):
             '-e', 'PYTHONUNBUFFERED=1',
         ]
 
+        # GPU or not
+        if os.environ.get("USE_GPU"):
+            engine_cmd.extend(['--gpus', 'all'])
+
         if kind == 'ingestion':
             # program here is either scoring program or submission, depends on if this ran during Prediction or Scoring
             if self.ingestion_only_during_scoring and self.is_scoring: