From 494f0d50fbb9ffec14fbb6791f47364f788d34db Mon Sep 17 00:00:00 2001 From: Ran Lu Date: Wed, 11 Sep 2024 13:49:17 -0400 Subject: [PATCH] Adjust airflow parameters Detect and reschedule tasks faster --- config/airflow.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/airflow.cfg b/config/airflow.cfg index 47c32fd2..bd42217b 100644 --- a/config/airflow.cfg +++ b/config/airflow.cfg @@ -459,14 +459,14 @@ print_stats_interval = 300 child_process_log_directory = /usr/local/airflow/logs/scheduler # How often (in seconds) should the scheduler check for orphaned tasks and SchedulerJobs -orphaned_tasks_check_interval = 600.0 +orphaned_tasks_check_interval = 300.0 # Local task jobs periodically heartbeat to the DB. If the job has # not heartbeat in this many seconds, the scheduler will mark the # associated task instance as failed and will re-schedule the task. -scheduler_zombie_task_threshold = 900 +scheduler_zombie_task_threshold = 120 -zombie_detection_interval = 180.0 +zombie_detection_interval = 60.0 # Turn off scheduler catchup by setting this to False. # Default behavior is unchanged and