diff --git a/playbooks/yarn_resourcemanager_capacityscheduler.yml b/playbooks/yarn_resourcemanager_capacityscheduler.yml index 133f6235..2f41f9aa 100644 --- a/playbooks/yarn_resourcemanager_capacityscheduler.yml +++ b/playbooks/yarn_resourcemanager_capacityscheduler.yml @@ -4,9 +4,10 @@ --- - name: Hadoop Yarn ResourceManager Capacity Scheduler hosts: yarn_rm + serial: 1 tasks: - tosit.tdp.resolve: # noqa unnamed-task - node_name: yarn_resourcemanager + node_name: yarn_scheduler - name: Configure YARN RM capacity scheduler ansible.builtin.import_role: name: tosit.tdp.yarn.resourcemanager diff --git a/playbooks/yarn_resourcemanager_config.yml b/playbooks/yarn_resourcemanager_config.yml index 63f31ea5..d5dc3565 100644 --- a/playbooks/yarn_resourcemanager_config.yml +++ b/playbooks/yarn_resourcemanager_config.yml @@ -12,3 +12,15 @@ name: tosit.tdp.yarn.resourcemanager tasks_from: config - ansible.builtin.meta: clear_facts # noqa unnamed-task + +- name: Hadoop Yarn ResourceManager Capacity Scheduler + hosts: yarn_rm + serial: 1 + tasks: + - tosit.tdp.resolve: # noqa unnamed-task + node_name: yarn_scheduler + - name: Configure YARN RM capacity scheduler + ansible.builtin.import_role: + name: tosit.tdp.yarn.resourcemanager + tasks_from: capacity_scheduler + - ansible.builtin.meta: clear_facts # noqa unnamed-task diff --git a/roles/yarn/resourcemanager/tasks/capacity_scheduler.yml b/roles/yarn/resourcemanager/tasks/capacity_scheduler.yml index ac84d94c..89d8ae2b 100644 --- a/roles/yarn/resourcemanager/tasks/capacity_scheduler.yml +++ b/roles/yarn/resourcemanager/tasks/capacity_scheduler.yml @@ -10,7 +10,10 @@ group: root mode: "644" -- name: Restart YARN Resource Manager - ansible.builtin.service: - name: hadoop-yarn-resourcemanager - state: restarted +- name: "Yarn scheduler | take kerberos ticket" + ansible.builtin.command: kinit -kt /etc/security/keytabs/rm.service.keytab rm/{{ ansible_fqdn }}@{{ realm }} + become_user: yarn + +- name: "Yarn scheduler | Refresh Queues" + command: /usr/bin/yarn rmadmin -refreshQueues + become_user: yarn diff --git a/roles/yarn/resourcemanager/tasks/config.yml b/roles/yarn/resourcemanager/tasks/config.yml index 5d79c733..43893324 100644 --- a/roles/yarn/resourcemanager/tasks/config.yml +++ b/roles/yarn/resourcemanager/tasks/config.yml @@ -66,14 +66,6 @@ group: root mode: "644" -- name: Render capacity-scheduler.xml - ansible.builtin.template: - src: capacity-scheduler.xml.j2 - dest: "{{ hadoop_rm_conf_dir }}/capacity-scheduler.xml" - owner: root - group: root - mode: "644" - - name: Render jmxremote.password ansible.builtin.template: src: jmxremote.password.j2 diff --git a/tdp_vars_defaults/yarn/yarn.yml b/tdp_vars_defaults/yarn/yarn.yml index 9583a397..9673f0bb 100644 --- a/tdp_vars_defaults/yarn/yarn.yml +++ b/tdp_vars_defaults/yarn/yarn.yml @@ -156,27 +156,6 @@ ranger_yarn_install_properties: XAAUDIT_SOLR_ENABLE: "{% if 'ranger_solr' in groups and groups['ranger_solr'] %}true{% else %}false{% endif %}" XAAUDIT_SOLR_URL: "{% if 'ranger_solr' in groups and groups['ranger_solr'] %}http://{{ groups['ranger_solr'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ ranger_solr_http_port }}/solr/ranger_audits{% else %}NONE{% endif %}" -capacity_scheduler: - yarn.scheduler.capacity.maximum-applications: 10000 - yarn.scheduler.capacity.maximum-am-resource-percent: 0.1 - yarn.scheduler.capacity.resource-calculator: org.apache.hadoop.yarn.util.resource.DominantResourceCalculator - yarn.scheduler.capacity.root.queues: default - yarn.scheduler.capacity.root.default.capacity: 100 - yarn.scheduler.capacity.root.default.user-limit-factor: 1 - yarn.scheduler.capacity.root.default.maximum-capacity: 100 - yarn.scheduler.capacity.root.default.state: RUNNING - yarn.scheduler.capacity.root.default.acl_submit_applications: "*" - yarn.scheduler.capacity.root.default.acl_administer_queue: "*" - yarn.scheduler.capacity.root.default.acl_application_max_priority: "*" - yarn.scheduler.capacity.root.default.maximum-application-lifetime: -1 - yarn.scheduler.capacity.root.default.default-application-lifetime: -1 - yarn.scheduler.capacity.node-locality-delay: 40 - yarn.scheduler.capacity.rack-locality-additional-delay: -1 - yarn.scheduler.capacity.queue-mappings: "" - yarn.scheduler.capacity.queue-mappings-override.enable: "false" - yarn.scheduler.capacity.per-node-heartbeat.maximum-offswitch-assignments: 1 - yarn.scheduler.capacity.application.fail-fast: false - # YARN resources allocation yarn_resourcemanager_heapsize: 1024m yarn_nodemanager_heapsize: 1024m diff --git a/tdp_vars_defaults/yarn/yarn_scheduler.yml b/tdp_vars_defaults/yarn/yarn_scheduler.yml new file mode 100644 index 00000000..6f757255 --- /dev/null +++ b/tdp_vars_defaults/yarn/yarn_scheduler.yml @@ -0,0 +1,24 @@ +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +--- +capacity_scheduler: + yarn.scheduler.capacity.maximum-applications: 10000 + yarn.scheduler.capacity.maximum-am-resource-percent: 0.1 + yarn.scheduler.capacity.resource-calculator: org.apache.hadoop.yarn.util.resource.DominantResourceCalculator + yarn.scheduler.capacity.root.queues: default + yarn.scheduler.capacity.root.default.capacity: 100 + yarn.scheduler.capacity.root.default.user-limit-factor: 1 + yarn.scheduler.capacity.root.default.maximum-capacity: 100 + yarn.scheduler.capacity.root.default.state: RUNNING + yarn.scheduler.capacity.root.default.acl_submit_applications: "*" + yarn.scheduler.capacity.root.default.acl_administer_queue: "*" + yarn.scheduler.capacity.root.default.acl_application_max_priority: "*" + yarn.scheduler.capacity.root.default.maximum-application-lifetime: -1 + yarn.scheduler.capacity.root.default.default-application-lifetime: -1 + yarn.scheduler.capacity.node-locality-delay: 40 + yarn.scheduler.capacity.rack-locality-additional-delay: -1 + yarn.scheduler.capacity.queue-mappings: "" + yarn.scheduler.capacity.queue-mappings-override.enable: "false" + yarn.scheduler.capacity.per-node-heartbeat.maximum-offswitch-assignments: 1 + yarn.scheduler.capacity.application.fail-fast: false