Share a airflow-log-storage volume to store logs locally

This is needed to view tasks in up for retry states
seung-lab · Nov 22, 2023 · afce3fd · afce3fd
1 parent b5b00da
commit afce3fd
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 0 deletions.
diff --git a/deploy/docker-compose-CeleryExecutor.yml b/deploy/docker-compose-CeleryExecutor.yml
@@ -24,6 +24,7 @@ x-airflow-common:
     volumes:
         - /var/run/docker.sock:/var/run/docker.sock
         - /tmp:/tmp
+        - airflow-log-storage:${AIRFLOW__LOGGING__BASE_LOG_FOLDER}
     deploy:
         restart_policy:
             condition: any
@@ -35,6 +36,7 @@ volumes:
     grafana-storage:
     redis-storage:
     jupyterlab-storage:
+    airflow-log-storage:
 
 services:
     rabbitmq:

diff --git a/deploy/docker-compose-local.yml b/deploy/docker-compose-local.yml
@@ -27,6 +27,7 @@ x-airflow-common:
         # - /my/local/data:/path/to/mount
         - /var/run/docker.sock:/var/run/docker.sock
         - /tmp:/tmp
+        - airflow-log-storage:${AIRFLOW__LOGGING__BASE_LOG_FOLDER}
     deploy:
         restart_policy:
             condition: any

diff --git a/scripts/entrypoint-dood.sh b/scripts/entrypoint-dood.sh
@@ -10,6 +10,10 @@ echo "start script with group $DOCKER_GROUP"
 
 if [[ -z "${_AIRFLOW_DB_UPGRADE=}" ]] ; then
     python scripts/install_packages.py
+else
+    if [ ! -f .sudo.disabled  ]; then
+        sudo chown -R $AIRFLOW_USER $AIRFLOW__LOGGING__BASE_LOG_FOLDER
+    fi
 fi
 
 if [[ -n "${CONDA_INSTALL_PYTORCH=}" ]] ; then