diff --git a/README.md b/README.md index a158609..60f0627 100644 --- a/README.md +++ b/README.md @@ -86,10 +86,12 @@ force_singularity_jobs => true, singularity_image_expr => "/images/myimage.img", singularity_bind_paths => ['/some_shared_filesystem', '/pool', '/usr/libexec/condor/'], singularity_target_dir => '/srv', +starter_job_environment => { 'SINGULARITY_HOME' => '/srv' }, mount_under_scratch_dirs => ['/tmp','/var/tmp'], ``` This forces all jobs to run inside Singularity containers, while offering `tmp` space inside the container, and binding a shared filesystem mount point and HTCondor-specific directories inside. The binding of the two HTCondor specific directories is a workaround to allow interactive jobs to run, this will hopefully be fixed in a future HTCondor release. +The same holds for setting `SINGULARITY_HOME`: This ensures non-interactive jobs start in the job's working directory instead of the user's home directory which might not even be accessible from the worker. The Image may also be an expression to allow for user configuration, more details on that are provided in the [HTCondor documentation](https://research.cs.wisc.edu/htcondor/manual/latest/3_17Singularity_Support.html). diff --git a/manifests/config/worker.pp b/manifests/config/worker.pp index fa0cb1e..c1ee90b 100644 --- a/manifests/config/worker.pp +++ b/manifests/config/worker.pp @@ -17,6 +17,7 @@ $memory_overcommit = $htcondor::memory_overcommit $number_of_cpus = $htcondor::number_of_cpus $partitionable_slots = $htcondor::partitionable_slots + $starter_job_environment = $htcondor::starter_job_environment $pool_create = $htcondor::pool_create $pool_home = $htcondor::pool_home $use_pid_namespaces = $htcondor::use_pid_namespaces diff --git a/manifests/init.pp b/manifests/init.pp index 06e0f43..2593e33 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -137,6 +137,7 @@ $request_memory = $htcondor::params::request_memory, $certificate_mapfile = $htcondor::params::certificate_mapfile, $kerberos_mapfile = $htcondor::params::kerberos_mapfile, + $starter_job_environment = $htcondor::params::starter_job_environment, $pool_home = $htcondor::params::pool_home, $pool_create = $htcondor::params::pool_create, $mount_under_scratch_dirs = $htcondor::params::mount_under_scratch_dirs, @@ -190,21 +191,21 @@ $cert_map_file_source = $htcondor::params::cert_map_file_source, $krb_map_file = $htcondor::params::krb_map_file, $krb_map_file_source = $htcondor::params::krb_map_file_source, - $ssl_server_keyfile = $htcondor::ssl_server_keyfile, - $ssl_client_keyfile = $htcondor::ssl_client_keyfile, - $ssl_server_certfile = $htcondor::ssl_server_certfile, - $ssl_client_certfile = $htcondor::ssl_client_certfile, - $ssl_server_cafile = $htcondor::ssl_server_cafile, - $ssl_client_cafile = $htcondor::ssl_client_cafile, - $ssl_server_cadir = $htcondor::ssl_server_cadir, - $ssl_client_cadir = $htcondor::ssl_client_cadir, + $ssl_server_keyfile = $htcondor::params::ssl_server_keyfile, + $ssl_client_keyfile = $htcondor::params::ssl_client_keyfile, + $ssl_server_certfile = $htcondor::params::ssl_server_certfile, + $ssl_client_certfile = $htcondor::params::ssl_client_certfile, + $ssl_server_cafile = $htcondor::params::ssl_server_cafile, + $ssl_client_cafile = $htcondor::params::ssl_client_cafile, + $ssl_server_cadir = $htcondor::params::ssl_server_cadir, + $ssl_client_cadir = $htcondor::params::ssl_client_cadir, $machine_list_prefix = $htcondor::params::machine_list_prefix, $max_walltime = $htcondor::params::max_walltime, $max_cputime = $htcondor::params::max_cputime, - $memory_factor = $htcondor::paramse::memory_factor, - $use_shared_port = $htcondor::use_shared_port, - $shared_port = $htcondor::shared_port, - $shared_port_collector_name = $htcondor::shared_port_collector_name, + $memory_factor = $htcondor::params::memory_factor, + $use_shared_port = $htcondor::params::use_shared_port, + $shared_port = $htcondor::params::shared_port, + $shared_port_collector_name = $htcondor::params::shared_port_collector_name, $use_singularity = $htcondor::params::use_singularity, $singularity_path = $htcondor::params::singularity_path, $force_singularity_jobs = $htcondor::params::force_singularity_jobs, diff --git a/manifests/params.pp b/manifests/params.pp index dd339c2..0123b3b 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -72,6 +72,7 @@ $memory_overcommit = hiera('memory_overcommit', 1.5) $request_memory = hiera('request_memory', true) + $starter_job_environment = hiera_hash('starter_job_environment', {}) $pool_home = hiera('pool_home', '/pool') $pool_create = hiera('pool_create', true) $mount_under_scratch_dirs = hiera_array('mount_under_scratch_dirs', ['/tmp', '/var/tmp']) diff --git a/templates/20_workernode.config.erb b/templates/20_workernode.config.erb index 0fc885f..4e745a2 100644 --- a/templates/20_workernode.config.erb +++ b/templates/20_workernode.config.erb @@ -82,12 +82,15 @@ MAXJOBRETIREMENTTIME = $(HOUR) * 24 * 3 UPDATE_INTERVAL = $RANDOM_INTEGER(230, 370) MASTER_UPDATE_INTERVAL = $RANDOM_INTEGER(230, 370) +## Special environment setup +STARTER_JOB_ENVIRONMENT = "<%= @starter_job_environment.map{|e| e.join('=')}.join(" ") %>" + ## Location of scratch directories EXECUTE = <%= @pool_home %>/condor ## Writable scratch directories bind mounted in scratch, e.g. for docker / singularity containers. ## Auto-deleted after the job exits. -MOUNT_UNDER_SCRATCH = "<%= @mount_under_scratch_dirs.flatten.join(", ") %>" +MOUNT_UNDER_SCRATCH = <%= @mount_under_scratch_dirs.flatten.join(", ") %> ## Make sure jobs have independent PID namespaces <% if @use_pid_namespaces -%>