feat: capacity scheduler separation

playbooks/yarn_resourcemanager_capacityscheduler.yml

@@ -4,9 +4,10 @@
 ---
 - name: Hadoop Yarn ResourceManager Capacity Scheduler
   hosts: yarn_rm
+  serial: 1
   tasks:
     - tosit.tdp.resolve: # noqa unnamed-task
-        node_name: yarn_resourcemanager
+        node_name: yarn_scheduler
     - name: Configure YARN RM capacity scheduler
       ansible.builtin.import_role:
         name: tosit.tdp.yarn.resourcemanager

playbooks/yarn_resourcemanager_config.yml

@@ -12,3 +12,15 @@
         name: tosit.tdp.yarn.resourcemanager
         tasks_from: config
     - ansible.builtin.meta: clear_facts # noqa unnamed-task
+
+- name: Hadoop Yarn ResourceManager Capacity Scheduler
+  hosts: yarn_rm
+  serial: 1
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: yarn_scheduler
+    - name: Configure YARN RM capacity scheduler
+      ansible.builtin.import_role:
+        name: tosit.tdp.yarn.resourcemanager
+        tasks_from: capacity_scheduler
+    - ansible.builtin.meta: clear_facts # noqa unnamed-task

roles/yarn/resourcemanager/tasks/capacity_scheduler.yml

@@ -10,7 +10,10 @@
     group: root
     mode: "644"
 
-- name: Restart YARN Resource Manager
-  ansible.builtin.service:
-    name: hadoop-yarn-resourcemanager
-    state: restarted
+- name: "Yarn scheduler | take kerberos ticket"
+  ansible.builtin.command: kinit -kt /etc/security/keytabs/rm.service.keytab rm/{{ ansible_fqdn }}@{{ realm }}
+  become_user: yarn
+
+- name: "Yarn scheduler | Refresh Queues"
+  command: /usr/bin/yarn rmadmin -refreshQueues
+  become_user: yarn

roles/yarn/resourcemanager/tasks/config.yml

@@ -66,14 +66,6 @@
     group: root
     mode: "644"
 
-- name: Render capacity-scheduler.xml
-  ansible.builtin.template:
-    src: capacity-scheduler.xml.j2
-    dest: "{{ hadoop_rm_conf_dir }}/capacity-scheduler.xml"
-    owner: root
-    group: root
-    mode: "644"
-
 - name: Render jmxremote.password
   ansible.builtin.template:
     src: jmxremote.password.j2

tdp_vars_defaults/yarn/yarn.yml

@@ -156,27 +156,6 @@ ranger_yarn_install_properties:
   XAAUDIT_SOLR_ENABLE: "{% if 'ranger_solr' in groups and groups['ranger_solr'] %}true{% else %}false{% endif %}"
   XAAUDIT_SOLR_URL: "{% if 'ranger_solr' in groups and groups['ranger_solr'] %}http://{{ groups['ranger_solr'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ ranger_solr_http_port }}/solr/ranger_audits{% else %}NONE{% endif %}"
 
-capacity_scheduler:
-  yarn.scheduler.capacity.maximum-applications: 10000
-  yarn.scheduler.capacity.maximum-am-resource-percent: 0.1
-  yarn.scheduler.capacity.resource-calculator: org.apache.hadoop.yarn.util.resource.DominantResourceCalculator
-  yarn.scheduler.capacity.root.queues: default
-  yarn.scheduler.capacity.root.default.capacity: 100
-  yarn.scheduler.capacity.root.default.user-limit-factor: 1
-  yarn.scheduler.capacity.root.default.maximum-capacity: 100
-  yarn.scheduler.capacity.root.default.state: RUNNING
-  yarn.scheduler.capacity.root.default.acl_submit_applications: "*"
-  yarn.scheduler.capacity.root.default.acl_administer_queue: "*"
-  yarn.scheduler.capacity.root.default.acl_application_max_priority: "*"
-  yarn.scheduler.capacity.root.default.maximum-application-lifetime: -1
-  yarn.scheduler.capacity.root.default.default-application-lifetime: -1
-  yarn.scheduler.capacity.node-locality-delay: 40
-  yarn.scheduler.capacity.rack-locality-additional-delay: -1
-  yarn.scheduler.capacity.queue-mappings: ""
-  yarn.scheduler.capacity.queue-mappings-override.enable: "false"
-  yarn.scheduler.capacity.per-node-heartbeat.maximum-offswitch-assignments: 1
-  yarn.scheduler.capacity.application.fail-fast: false
-
 # YARN resources allocation
 yarn_resourcemanager_heapsize: 1024m
 yarn_nodemanager_heapsize: 1024m

tdp_vars_defaults/yarn/yarn_scheduler.yml

@@ -0,0 +1,24 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+capacity_scheduler:
+  yarn.scheduler.capacity.maximum-applications: 10000
+  yarn.scheduler.capacity.maximum-am-resource-percent: 0.1
+  yarn.scheduler.capacity.resource-calculator: org.apache.hadoop.yarn.util.resource.DominantResourceCalculator
+  yarn.scheduler.capacity.root.queues: default
+  yarn.scheduler.capacity.root.default.capacity: 100
+  yarn.scheduler.capacity.root.default.user-limit-factor: 1
+  yarn.scheduler.capacity.root.default.maximum-capacity: 100
+  yarn.scheduler.capacity.root.default.state: RUNNING
+  yarn.scheduler.capacity.root.default.acl_submit_applications: "*"
+  yarn.scheduler.capacity.root.default.acl_administer_queue: "*"
+  yarn.scheduler.capacity.root.default.acl_application_max_priority: "*"
+  yarn.scheduler.capacity.root.default.maximum-application-lifetime: -1
+  yarn.scheduler.capacity.root.default.default-application-lifetime: -1
+  yarn.scheduler.capacity.node-locality-delay: 40
+  yarn.scheduler.capacity.rack-locality-additional-delay: -1
+  yarn.scheduler.capacity.queue-mappings: ""
+  yarn.scheduler.capacity.queue-mappings-override.enable: "false"
+  yarn.scheduler.capacity.per-node-heartbeat.maximum-offswitch-assignments: 1
+  yarn.scheduler.capacity.application.fail-fast: false