limit concurrent_task to executor_cores

milenkovicm · Nov 26, 2024 · 6e0407b · 6e0407b
1 parent 74ca8dd
commit 6e0407b
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/ballista/executor/executor_config_spec.toml b/ballista/executor/executor_config_spec.toml
@@ -75,7 +75,6 @@ doc = "Directory for temporary IPC files"
 abbr = "c"
 name = "concurrent_tasks"
 type = "usize"
-default = "0" # defaults to all available cores if left as zero
 doc = "Max concurrent tasks."
 
 [[param]]

diff --git a/ballista/executor/src/config.rs b/ballista/executor/src/config.rs
@@ -35,6 +35,11 @@ impl TryFrom<Config> for ExecutorProcessConfig {
             opt.bind_port
         );
 
+        let concurrent_tasks = opt
+            .concurrent_tasks
+            .or(opt.executor_cores)
+            .unwrap_or_else(|| std::thread::available_parallelism().unwrap().get());
+
         Ok(ExecutorProcessConfig {
             special_mod_log_level: opt.log_level_setting,
             external_host: opt.external_host,
@@ -44,7 +49,7 @@ impl TryFrom<Config> for ExecutorProcessConfig {
             scheduler_host: opt.scheduler_host,
             scheduler_port: opt.scheduler_port,
             scheduler_connect_timeout_seconds: opt.scheduler_connect_timeout_seconds,
-            concurrent_tasks: opt.concurrent_tasks,
+            concurrent_tasks,
             task_scheduling_policy: opt.task_scheduling_policy,
             work_dir: opt.work_dir,
             log_dir: opt.log_dir,