From 51f84e18bf1492391b0f525e20f8991a664c1c57 Mon Sep 17 00:00:00 2001 From: Oliver Freyermuth Date: Thu, 7 Sep 2017 13:48:59 +0200 Subject: [PATCH 1/4] Remove bad quotes from MOUNT_UNDER_SCRATCH variable. This is not an expression, so the additional quotes are interpreted as actual part of the path. --- templates/20_workernode.config.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/20_workernode.config.erb b/templates/20_workernode.config.erb index 0fc885f..6d46b2e 100644 --- a/templates/20_workernode.config.erb +++ b/templates/20_workernode.config.erb @@ -87,7 +87,7 @@ EXECUTE = <%= @pool_home %>/condor ## Writable scratch directories bind mounted in scratch, e.g. for docker / singularity containers. ## Auto-deleted after the job exits. -MOUNT_UNDER_SCRATCH = "<%= @mount_under_scratch_dirs.flatten.join(", ") %>" +MOUNT_UNDER_SCRATCH = <%= @mount_under_scratch_dirs.flatten.join(", ") %> ## Make sure jobs have independent PID namespaces <% if @use_pid_namespaces -%> From 850e21de2e6cc8bc089eb071c6827528cef16517 Mon Sep 17 00:00:00 2001 From: Oliver Freyermuth Date: Fri, 8 Sep 2017 15:31:32 +0200 Subject: [PATCH 2/4] init: Fixup parameter default values. Sometimes, params-namespace was missing, and once it was mis-spelled (for memory_factor). --- manifests/init.pp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/manifests/init.pp b/manifests/init.pp index c0bc5c9..035cc2d 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -182,7 +182,7 @@ $use_claim_to_be_auth = $htcondor::params::use_claim_to_be_auth, $use_cert_map_file = $htcondor::params::use_cert_map_file, $use_krb_map_file = $htcondor::params::use_krb_map_file, - $use_ssl_auth = $htcondor::use_ssl_auth, + $use_ssl_auth = $htcondor::params::use_ssl_auth, $use_pid_namespaces = $htcondor::params::use_pid_namespaces, $uses_connection_broker = $htcondor::params::uses_connection_broker, $private_network_name = $htcondor::params::private_network_name, @@ -190,21 +190,21 @@ $cert_map_file_source = $htcondor::params::cert_map_file_source, $krb_map_file = $htcondor::params::krb_map_file, $krb_map_file_source = $htcondor::params::krb_map_file_source, - $ssl_server_keyfile = $htcondor::ssl_server_keyfile, - $ssl_client_keyfile = $htcondor::ssl_client_keyfile, - $ssl_server_certfile = $htcondor::ssl_server_certfile, - $ssl_client_certfile = $htcondor::ssl_client_certfile, - $ssl_server_cafile = $htcondor::ssl_server_cafile, - $ssl_client_cafile = $htcondor::ssl_client_cafile, - $ssl_server_cadir = $htcondor::ssl_server_cadir, - $ssl_client_cadir = $htcondor::ssl_client_cadir, + $ssl_server_keyfile = $htcondor::params::ssl_server_keyfile, + $ssl_client_keyfile = $htcondor::params::ssl_client_keyfile, + $ssl_server_certfile = $htcondor::params::ssl_server_certfile, + $ssl_client_certfile = $htcondor::params::ssl_client_certfile, + $ssl_server_cafile = $htcondor::params::ssl_server_cafile, + $ssl_client_cafile = $htcondor::params::ssl_client_cafile, + $ssl_server_cadir = $htcondor::params::ssl_server_cadir, + $ssl_client_cadir = $htcondor::params::ssl_client_cadir, $machine_list_prefix = $htcondor::params::machine_list_prefix, $max_walltime = $htcondor::params::max_walltime, $max_cputime = $htcondor::params::max_cputime, - $memory_factor = $htcondor::paramse::memory_factor, - $use_shared_port = $htcondor::use_shared_port, - $shared_port = $htcondor::shared_port, - $shared_port_collector_name = $htcondor::shared_port_collector_name, + $memory_factor = $htcondor::params::memory_factor, + $use_shared_port = $htcondor::params::use_shared_port, + $shared_port = $htcondor::params::shared_port, + $shared_port_collector_name = $htcondor::params::shared_port_collector_name, $use_singularity = $htcondor::params::use_singularity, $singularity_path = $htcondor::params::singularity_path, $force_singularity_jobs = $htcondor::params::force_singularity_jobs, From 17d134a4ba0aa54d92c8691169d75492eee77387 Mon Sep 17 00:00:00 2001 From: Oliver Freyermuth Date: Wed, 20 Sep 2017 17:13:19 +0200 Subject: [PATCH 3/4] Add configuration of STARTER_JOB_ENVIRONMENT. This allows to pass on environment seen by the user's jobs, and also e.g. by the executed singularity container. --- manifests/config/worker.pp | 1 + manifests/init.pp | 1 + manifests/params.pp | 1 + templates/20_workernode.config.erb | 3 +++ 4 files changed, 6 insertions(+) diff --git a/manifests/config/worker.pp b/manifests/config/worker.pp index 663ee71..9998fa2 100644 --- a/manifests/config/worker.pp +++ b/manifests/config/worker.pp @@ -17,6 +17,7 @@ $memory_overcommit = $htcondor::memory_overcommit $number_of_cpus = $htcondor::number_of_cpus $partitionable_slots = $htcondor::partitionable_slots + $starter_job_environment = $htcondor::starter_job_environment $pool_create = $htcondor::pool_create $pool_home = $htcondor::pool_home $use_pid_namespaces = $htcondor::use_pid_namespaces diff --git a/manifests/init.pp b/manifests/init.pp index 035cc2d..2593e33 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -137,6 +137,7 @@ $request_memory = $htcondor::params::request_memory, $certificate_mapfile = $htcondor::params::certificate_mapfile, $kerberos_mapfile = $htcondor::params::kerberos_mapfile, + $starter_job_environment = $htcondor::params::starter_job_environment, $pool_home = $htcondor::params::pool_home, $pool_create = $htcondor::params::pool_create, $mount_under_scratch_dirs = $htcondor::params::mount_under_scratch_dirs, diff --git a/manifests/params.pp b/manifests/params.pp index ba4f6ea..ec4cc19 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -72,6 +72,7 @@ $memory_overcommit = hiera('memory_overcommit', 1.5) $request_memory = hiera('request_memory', true) + $starter_job_environment = hiera_hash('starter_job_environment', {}) $pool_home = hiera('pool_home', '/pool') $pool_create = hiera('pool_create', true) $mount_under_scratch_dirs = hiera_array('mount_under_scratch_dirs', ['/tmp', '/var/tmp']) diff --git a/templates/20_workernode.config.erb b/templates/20_workernode.config.erb index 6d46b2e..4e745a2 100644 --- a/templates/20_workernode.config.erb +++ b/templates/20_workernode.config.erb @@ -82,6 +82,9 @@ MAXJOBRETIREMENTTIME = $(HOUR) * 24 * 3 UPDATE_INTERVAL = $RANDOM_INTEGER(230, 370) MASTER_UPDATE_INTERVAL = $RANDOM_INTEGER(230, 370) +## Special environment setup +STARTER_JOB_ENVIRONMENT = "<%= @starter_job_environment.map{|e| e.join('=')}.join(" ") %>" + ## Location of scratch directories EXECUTE = <%= @pool_home %>/condor From 3a7df5e975fefb7b7e40609366e1e321a9c0a61b Mon Sep 17 00:00:00 2001 From: Oliver Freyermuth Date: Wed, 20 Sep 2017 18:37:39 +0200 Subject: [PATCH 4/4] README: Explain usecase of starter_job_environment with singularity. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a158609..60f0627 100644 --- a/README.md +++ b/README.md @@ -86,10 +86,12 @@ force_singularity_jobs => true, singularity_image_expr => "/images/myimage.img", singularity_bind_paths => ['/some_shared_filesystem', '/pool', '/usr/libexec/condor/'], singularity_target_dir => '/srv', +starter_job_environment => { 'SINGULARITY_HOME' => '/srv' }, mount_under_scratch_dirs => ['/tmp','/var/tmp'], ``` This forces all jobs to run inside Singularity containers, while offering `tmp` space inside the container, and binding a shared filesystem mount point and HTCondor-specific directories inside. The binding of the two HTCondor specific directories is a workaround to allow interactive jobs to run, this will hopefully be fixed in a future HTCondor release. +The same holds for setting `SINGULARITY_HOME`: This ensures non-interactive jobs start in the job's working directory instead of the user's home directory which might not even be accessible from the worker. The Image may also be an expression to allow for user configuration, more details on that are provided in the [HTCondor documentation](https://research.cs.wisc.edu/htcondor/manual/latest/3_17Singularity_Support.html).