HTCondor-CE 4.0 support and APEL accounting

.travis.yml

@@ -9,37 +9,10 @@ script: bundle exec rake validate && bundle exec rake lint && bundle exec rake s
 matrix:
   fast_finish: true
   include:
-  - rvm: 1.9.3
-    env: PUPPET_VERSION="~> 3.4.0"
-  - rvm: 1.9.3
-    env: PUPPET_VERSION="~> 3.0"
-  - rvm: 1.9.3
-    env: PUPPET_VERSION="~> 3.0" FUTURE_PARSER="yes"
-  - rvm: 1.9.3
-    env: PUPPET_VERSION="~> 4.0"
-  - rvm: 2.0.0
-    env: PUPPET_VERSION="~> 3.0"
-  - rvm: 2.0.0
-    env: PUPPET_VERSION="~> 3.0" FUTURE_PARSER="yes"
-  - rvm: 2.0.0
-    env: PUPPET_VERSION="~> 4.0"
-  - rvm: 2.1.5
-    env: PUPPET_VERSION="~> 3.0"
-  - rvm: 2.1.5
-    env: PUPPET_VERSION="~> 3.0" FUTURE_PARSER="yes"
-  - rvm: 2.1.5
-    env: PUPPET_VERSION="~> 4.0"
-  - rvm: 2.1.5
-    env: PUPPET_VERSION="~> 4.0" STRICT_VARIABLES="yes"
-  - rvm: 2.2.3
-    env: PUPPET_VERSION="~> 4.0"
-  - rvm: default
-    sudo: required
-    dist: trusty
-    services: docker
-    env: BEAKER_set="centos-6-x86_64-docker"
-    bundler_args:
-    script: sudo service docker restart ; sleep 10 && bundle exec rake beaker
+  - rvm: 2.4.5
+    env: PUPPET_GEM_VERSION="~> 5.0"
+  - rvm: 2.5.3
+    env: PUPPET_GEM_VERSION="~> 6.0"
   - rvm: default
     sudo: required
     dist: trusty

README.md

@@ -14,6 +14,7 @@
 1. [Reference - An under-the-hood peek at what the module is doing and how](#reference)
 1. [Limitations - OS compatibility, etc.](#limitations)
 1. [Development - Guide for contributing to the module](#development)
+1. [GridPP Documentation](#GridPP-Documentation)
 
 ## Description
 
@@ -84,3 +85,22 @@ know what the ground rules for contributing are.
 If you aren't using changelog, put your release notes here (though you should
 consider using changelog). You can also add any additional sections you feel
 are necessary or important to include here. Please use the `## ` header.
+
+
+## GridPP Documentation
+
+- https://www.gridpp.ac.uk/wiki/Example_Build_of_an_HTCondor-CE_Cluster
+- https://twiki.cern.ch/twiki/bin/view/LCG/HtCondorCeAccounting
+
+## Defining job routes
+
+```yaml
+htcondor_ce::job_routes:
+  dice_centos7:
+    TargetUniverse: 5
+    eval_set_AccountingGroup: 'strcat("group_u_", x509userproxyvoname, ".", Owner)'
+    delete_SUBMIT_Iwd: true
+    set_WantIOProxy: true
+    set_default_maxMemory: 3000
+    set_Requirements: 'TARGET.OpSysAndVer == "CentOS7"'
+```

Rakefile

@@ -2,6 +2,7 @@ require 'rubygems'
 require 'puppetlabs_spec_helper/rake_tasks'
 require 'puppet-lint/tasks/puppet-lint'
 PuppetLint.configuration.send('disable_80chars')
+PuppetLint.configuration.send('disable_autoloader_layout')
 PuppetLint.configuration.ignore_paths = ["spec/**/*.pp", "pkg/**/*.pp"]
 
 desc "Validate manifests, templates, and ruby files"

files/01-ce-auth.conf

@@ -0,0 +1,20 @@
+###############################################################################
+#
+# HTCondor-CE common authorization configuration
+#
+# This file will NOT be overwritten upon RPM upgrade.
+#
+###############################################################################
+
+# Uncomment the following lines if your SciTokens or SSL clients use
+# grid certificates or your HTCondor-CE's host certificate is located
+# in the standard grid location
+#
+# https://htcondor-ce.readthedocs.io/en/latest/installation/htcondor-ce#configuring-certificates
+
+AUTH_SSL_SERVER_CERTFILE = /etc/grid-security/hostcert.pem
+AUTH_SSL_SERVER_KEYFILE = /etc/grid-security/hostkey.pem
+AUTH_SSL_SERVER_CADIR = /etc/grid-security/certificates
+AUTH_SSL_CLIENT_CADIR = /etc/grid-security/certificates
+# AUTH_SSL_SERVER_CAFILE =
+# AUTH_SSL_CLIENT_CAFILE =

files/apel/cron_condor-ce_apel.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# accountingRun.sh
+# [email protected], 2019
+# modified by [email protected]
+# 210331 modified by WL as per GGUS ticket #150995: "you need to run
+# separately the scripts apelclient and ssmsend, so the first one will
+# create the accounting records and the second one will send them."
+# Run the processes of a HTCondor accounting run
+
+/usr/share/condor-ce/condor_blah.sh    # Make the blah file (CE/Security data)
+/usr/share/condor-ce/condor_batch.sh      # Make the batch file (batch system job run times)
+/usr/bin/apelparser                # Read the blah and batch files in
+/usr/bin/apelclient                # Join blah and batch records to make job records
+/usr/bin/ssmsend                   # Send job records into APEL system

manifests/auth.pp

@@ -15,4 +15,10 @@
   } else {
     fail("This module currently doesn't support backends other than ARGUS.")
   }
+
+  package {['ca-certificates']:
+    ensure => latest,
+  }
+
+  include fetchcrl
 }

manifests/auth/argus.pp

@@ -20,29 +20,39 @@
   validate_integer($argus_port)
   validate_string($argus_resourceid)
 
-  package { 'argus-pep-api-c': ensure => present, }
-
-  package { 'argus-gsi-pep-callout':
-    ensure  => present,
-    require => Package['argus-pep-api-c'],
+  $argus_packages = [
+    'argus-gsi-pep-callout',
+    'lcas-plugins-basic',
+    'lcas-plugins-voms',
+    'lcmaps-plugins-basic',
+    'lcmaps-plugins-c-pep',
+    'lcmaps-plugins-verify-proxy',
+    'lcmaps-plugins-voms',
+  ]
+  package { $argus_packages:
+    ensure          => present,
+    install_options => ['--enablerepo', 'epel,wlcg,UMD-4-base,UMD-4-updates']
   }
-
-  file { $pep_callout:
+  -> file { $pep_callout:
     ensure  => file,
     owner   => 'root',
     group   => 'root',
     mode    => '0644',
     content => template("${module_name}/gsi-pep-callout.erb"),
     require => Package['argus-gsi-pep-callout'],
-  }
-
-  file { $gsi_authz:
+  } -> file { $gsi_authz:
     ensure  => file,
     owner   => 'root',
     group   => 'root',
     mode    => '0644',
     source  => "puppet:///modules/${module_name}/gsi-authz.conf",
     require => Package['argus-gsi-pep-callout'],
+  } -> file{ '/etc/lcmaps/lcmaps.db':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template("${module_name}/lcmaps.db.erb"),
+    require => Package['argus-gsi-pep-callout'],
   }
-
 }

manifests/config.pp

@@ -7,16 +7,16 @@
 class htcondor_ce::config {
   $site_security       = '/etc/condor-ce/config.d/59-site-security.conf'
   $main_ce_config      = '/etc/condor-ce/config.d/60-configured-attributes.conf'
-  $job_routes          = '/etc/condor-ce/config.d/61-job-routes.conf'
   $condor_mapfile      = '/etc/condor-ce/condor_mapfile'
   $ce_sysconfig        = '/etc/sysconfig/condor-ce'
   # general parameters used in manifest or more than one template
+  $install_apel        = $::htcondor_ce::install_apel
   $install_bdii        = $::htcondor_ce::install_bdii
-  $job_routes_template = $::htcondor_ce::job_routes_template
   $uid_domain          = $::htcondor_ce::uid_domain
   $use_static_shadow   = $::htcondor_ce::use_static_shadow
   # $site_security
   $gsi_regex           = $::htcondor_ce::gsi_regex
+  $host_dn             = $::htcondor_ce::host_dn
   # $main_ce_config
   $condor_view_hosts   = $::htcondor_ce::condor_view_hosts
   $pool_collectors     = $::htcondor_ce::pool_collectors
@@ -38,13 +38,7 @@
     content => template("${module_name}/60-configured-attributes.conf.erb"),
   }
 
-  file { $job_routes:
-    ensure  => file,
-    owner   => 'condor',
-    group   => 'condor',
-    mode    => '0644',
-    content => template($job_routes_template),
-  }
+  class {'::htcondor_ce::config::job_routes': }
 
   file { $condor_mapfile:
     ensure  => file,
@@ -62,7 +56,7 @@
     source => "puppet:///modules/${module_name}/sysconfig-condor-ce",
   }
 
-  $config_files = [File[$main_ce_config], File[$site_security], File[$job_routes], File[$condor_mapfile]]
+  $config_files = [File[$main_ce_config], File[$site_security], File[$condor_mapfile]]
 
   exec { '/usr/bin/condor_ce_reconfig': refreshonly => true, }
 
@@ -76,4 +70,10 @@
     class { '::htcondor_ce::config::shadow': }
   }
 
+  if $install_apel {
+    class { '::htcondor_ce::config::apel': }
+  }
+
+  class { '::htcondor_ce::config::certs': }
+
 }

manifests/config/apel.pp

@@ -0,0 +1,90 @@
+# class htcondor::config::apel
+#
+# This class configures the
+# APEL Accounting for the HTCondor CE
+#
+# from https://twiki.cern.ch/twiki/bin/view/LCG/HtCondorCeAccounting
+class htcondor_ce::config::apel{
+  $apel_ce_config      = $::htcondor_ce::apel_ce_config
+  $apel_condor_config  = $::htcondor_ce::apel_condor_config
+  $apel_output_dir     = $::htcondor_ce::apel_output_dir
+  $apel_scaling_attr   = $::htcondor_ce::apel_scaling_attr
+  $apel_mysql_root_pw  = $::htcondor_ce::apel_mysql_root_pw
+  $apel_mysql_db       = $::htcondor_ce::apel_mysql_db
+  $apel_mysql_user     = $::htcondor_ce::apel_mysql_user
+  $apel_mysql_password = $::htcondor_ce::apel_mysql_password
+  $apel_enable_ssm     = $::htcondor_ce::apel_enable_ssm
+  $hepspec06           = $::htcondor_ce::hepspec06
+
+  $goc_site_name       = $::htcondor_ce::goc_site_name
+
+
+  file{$apel_ce_config:
+    ensure  => present,
+    owner   => 'condor',
+    group   => 'condor',
+    mode    => '0644',
+    content => template("${module_name}/51-ce-apel.conf.erb"),
+    require => Package['htcondor-ce-apel'],
+  }
+
+  file{$apel_condor_config:
+    ensure  => present,
+    owner   => 'condor',
+    group   => 'condor',
+    mode    => '0644',
+    content => template("${module_name}/51-condor-apel.conf.erb"),
+    require => Package['htcondor-ce-apel'],
+  }
+
+  ## MySQL settings (apelparser)
+  class { '::mysql::server':
+    root_password           => $mysql_root_pw,
+    remove_default_accounts => true,
+    restart                 => true
+  }
+
+  mysql::db { $apel_mysql_db:
+    user     => $apel_mysql_user,
+    password => $apel_mysql_password,
+    host     => 'localhost',
+    grant    => ['ALL'],
+    sql      => '/usr/share/apel/client.sql'
+  }
+
+  file{'/etc/apel/client.cfg':
+    ensure  => present,
+    content => template("${module_name}/apel/client.cfg.erb"),
+    require => Package['htcondor-ce-apel'],
+  }
+
+  file{'/etc/apel/parser.cfg':
+    ensure  => present,
+    content => template("${module_name}/apel/parser.cfg.erb"),
+    require => Package['htcondor-ce-apel'],
+  }
+
+  file{'/etc/apel/sender.cfg':
+    ensure  => present,
+    content => template("${module_name}/apel/sender.cfg.erb"),
+    require => Package['htcondor-ce-apel'],
+  }
+
+  file{'/etc/apel/cron_condor-ce_apel.sh':
+    ensure => present,
+    source => "puppet:///modules/${module_name}/apel/cron_condor-ce_apel.sh",
+    mode   => '0755',
+  }
+
+  class { '::cron':
+    manage_package => false,
+  }
+
+  cron::job {'apel-processing':
+    minute      => '31',
+    hour        => '2',
+    command     => '/etc/apel/cron_condor-ce_apel.sh',
+    description => 'APEL HTCondor-CE job parsing',
+    require     => File['/etc/apel/cron_condor-ce_apel.sh'],
+  }
+}

manifests/config/bdii.pp

@@ -1,6 +1,6 @@
-# Class: htcondor_ce::bdii
+# Class: htcondor_ce::config bdii
 #
-# This class installs and configures the
+# This class configures the
 # Resource BDII for the HTCondor Computing Element
 #
 class htcondor_ce::config::bdii {
@@ -10,6 +10,7 @@
   $supported_vos       = $::htcondor_ce::supported_vos
   $goc_site_name       = $::htcondor_ce::goc_site_name
   $benchmark_result    = $::htcondor_ce::benchmark_result
+  $hepspec06           = $::htcondor_ce::hepspec06
   $execution_env_cores = $::htcondor_ce::execution_env_cores
   $election_type       = $::htcondor_ce::election_type
   $election_hosts      = $::htcondor_ce::election_hosts
@@ -25,4 +26,12 @@
 
   File[$bdii_ce_config] ~> Exec['/usr/bin/condor_ce_reconfig']
 
+  service{'bdii':
+    ensure  => running,
+    enable => true,
+  }
+  # make sure /etc/condor/config.d/99-ce-bdii.conf is removed
+  file {'/etc/condor/config.d/99-ce-bdii.conf':
+    ensure => absent,
+  }
 }

manifests/config/certs.pp

@@ -0,0 +1,38 @@
+# Configures host certificate and key files
+class htcondor_ce::config::certs{
+  $hostcert_location       = $::htcondor_ce::hostcert_location
+  $hostkey_location       = $::htcondor_ce::hostkey_location
+
+
+  file {'/etc/grid-security/hostcert.pem':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0444',
+    source  => $hostcert_location,
+  }
+
+  file {'/etc/grid-security/hostkey.pem':
+    ensure  => file,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0400',
+    source  => $hostkey_location,
+  }
+
+  file {'/etc/grid-security/condorcert.pem':
+    ensure  => file,
+    owner   => 'condor',
+    group   => 'condor',
+    mode    => '0444',
+    source  => $hostcert_location,
+  }
+
+  file {'/etc/grid-security/condorkey.pem':
+    ensure  => file,
+    owner   => 'condor',
+    group   => 'condor',
+    mode    => '0400',
+    source  => $hostkey_location,
+  }
+}