Refactored to use best practises

README.md

@@ -4,7 +4,7 @@ Ansible project to automatically configure APT-based hosts to act as master/slav
 
 *I should caveat the above by saying that I've only tested the scripts against Ubuntu v14.04. If you discover any problems using the playbook be sure to issue a pull-request.*
 
-## Usage
+## Creating a cluster
 
 To configure your Ubuntu machines as an MPI cluster first create an [inventory file](http://docs.ansible.com/ansible/intro_inventory.html) matching the template:
 
@@ -24,14 +24,19 @@ cd ./provisioning
 ansible-playbook site.yml -i <PATH TO INVENTORY>
 ```
 
+### Adding a new slave to the cluster
+
+If you want to add a slave to an existing cluster do the following:
+
+1. Add the slave's IP to the `slaves` group in your inventory file.
+2. Run `ansible-playbook slaves.yml -i <PATH TO INVENTORY>`
+
 ## Development
 
 [Vagrant](https://www.vagrantup.com/) is used to create development VMs against which you can test changes to the Ansible scripts.
 
 1. [Install Vagrant](https://www.vagrantup.com/docs/installation/).
-
-2. Provision and configure
-the local cluster with:
+2. Provision and configure the local cluster with:
 
 ```bash
 vagrant up

Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 
 VAGRANTFILE_API_VERSION = "2"
-TOTAL_SLAVES = 1
+TOTAL_SLAVES = 2
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 

provisioning/group_vars/all.yml

@@ -1,7 +1,7 @@
 ---
 
 mpi_user: mpiuser
-mpi_data: /srv/mpi_data
-mpi_data_master_public_key_file: "{{ mpi_data }}/.id_rsa.pub"
+mpi_data_share: /srv/mpi_data_share
+shared_master_public_key_file_path: "{{ mpi_data_share }}/.id_rsa.pub"
 
 mpi_hosts_file: /etc/mpi_hosts

provisioning/master.yml

@@ -0,0 +1,11 @@
+---
+
+- hosts: all
+  become: yes
+  tasks:
+    - include: tasks/create_hosts_file.yml
+
+- hosts: master
+  become: yes
+  roles:
+     - mpi_master

provisioning/roles/mpi/tasks/main.yml → provisioning/roles/mpi_base/tasks/main.yml

@@ -1,10 +1,9 @@
 ---
 
-- name: Install OpenMPI
+- name: Ensure OpenMPI present
   apt:
     name: "{{ item }}"
     state: present
-    update_cache: yes
   with_items:
     - libopenmpi-dev
     - openmpi-bin

provisioning/roles/mpi_master/meta/main.yml

@@ -1,4 +1,5 @@
 ---
 
 dependencies:
-  - { role: geerlingguy.nfs, nfs_exports: [ "{{ mpi_data }} *(ro,sync,no_root_squash)" ] }
+  - { role: mpi_base }
+  - { role: geerlingguy.nfs, nfs_exports: [ "{{ mpi_data_share }} *(ro,sync,no_root_squash)" ] }

provisioning/roles/mpi_master/tasks/main.yml

@@ -1,40 +1,51 @@
 ---
 
-- name: Create MPI user
+- name: Ensure MPI user exists with public/private key
   user:
     name: "{{ mpi_user }}"
     generate_ssh_key: yes
     ssh_key_bits: 2048
     ssh_key_file: .ssh/id_rsa
 
-- name: Save public key to NFS share
-  shell: "cp ~{{ mpi_user }}/.ssh/id_rsa.pub {{ mpi_data_master_public_key_file }}"
+- name: Read MPI user's public key
+  slurp:
+    src: "~{{ mpi_user }}/.ssh/id_rsa.pub"
+  register: public_key
 
-- name: Give MPI user access to NFS share
+- name: Ensure MPI user's public key is available to slaves in NFS share
+  blockinfile:
+    dest: "{{ shared_master_public_key_file_path }}"
+    block: "{{ public_key['content'] | b64decode }}"
+    state: present
+    create: yes
+
+- name: Ensure MPI user has permission to access NFS share
   file:
-    path: "{{ mpi_data }}"
+    path: "{{ mpi_data_share }}"
     state: directory
     recurse: yes
     owner: "{{ mpi_user }}"
     group: "{{ mpi_user }}"
     mode: 0755
 
-- name: Generate public ssh host keys for slave nodes so MPI can connect without confirmation
+- name: Read slave SSH host keys
   shell: ssh-keyscan {{ item }}
   with_items: "{{ groups['slaves'] }}"
   register: keyscan_results
+  changed_when: False
 
-- name: Write generated SSH host keys to known_hosts file
+- name: Ensure master's known_hosts contains slave SSH host keys for connections without confirmations
   lineinfile:
     dest: "~{{ mpi_user }}/.ssh/known_hosts"
-    line: "{{ item.stdout }}"
+    line: "{{ item }}"
+    regexp: "^{{ item | regex_replace('^([^ ]+ [^ ]+) .*', '\\1') }}"
     state: present
     create: yes
     owner: "{{ mpi_user }}"
     group: "{{ mpi_user }}"
-  with_items: "{{ keyscan_results.results }}"
+  with_items: "{{ keyscan_results.results | map(attribute='stdout_lines') | list }}"
 
-- name: Build MPI hosts file listing all nodes to be used in cluster
+- name: Ensure all hosts listed in MPI hosts file with slot availability
   lineinfile:
     dest: "{{ mpi_hosts_file }}"
     line: "{{ item }} slots=1"

provisioning/roles/mpi_slave/meta/main.yml

@@ -0,0 +1,4 @@
+---
+
+dependencies:
+  - { role: mpi_base }

provisioning/roles/mpi_slave/tasks/main.yml

@@ -1,38 +1,37 @@
 ---
 
-- name: Mount shared directory
+- name: Ensure NFS share mounted
   mount:
-    name: "{{ mpi_data }}"
-    src: "{{ master_hostname }}:{{ mpi_data }}"
+    name: "{{ mpi_data_share }}"
+    src: "{{ master_hostname }}:{{ mpi_data_share }}"
     fstype: nfs
     state: mounted
 
-- name: Create MPI user
+- name: Ensure MPI user exists
   user:
     name: "{{ mpi_user }}"
 
-- name: Give MPI user access to NFS share
+- name: Ensure MPI user has permission to access NFS share
   file:
-    path: "{{ mpi_data }}"
+    path: "{{ mpi_data_share }}"
     state: directory
     owner: "{{ mpi_user }}"
     group: "{{ mpi_user }}"
     mode: 0755
 
-
-- name: Read Master's public key
+- name: Read Master's public key from NFS share
   slurp:
-    src: "{{ mpi_data_master_public_key_file }}"
+    src: "{{ shared_master_public_key_file_path }}"
   register: slurpfile
 
-- name: Add Master's public key as authorized key
+- name: Ensure Master's public key is added as authorized key for connecting as MPI user
   authorized_key:
     user: "{{ mpi_user }}"
     state: present
     key: "{{ slurpfile['content'] | b64decode }}"
 
 # Module above will create authorized_keys file as root user
-- name: Give MPI user permission to access authorized_keys
+- name: Ensure MPI user can access its authorized_keys file
   file:
     path: "~{{ mpi_user }}/.ssh/authorized_keys"
     owner: "{{ mpi_user }}"

provisioning/site.yml

@@ -8,25 +8,11 @@
 
 - hosts: all
   become: yes
-  roles:
-    - mpi
   tasks:
-    - include: hosts.yml
+    - name: Ensure APT caches updated
+      apt:
+        update_cache: yes
 
-- hosts: master
-  become: yes
-  roles:
-     - mpi_master
-
-- hosts: slaves
-  become: yes
-  roles:
-     - mpi_slave
-
-- hosts: master
-  become: yes
-  vars:
-    mpi_tutorial_repo: 'https://github.com/wesleykendall/mpitutorial.git'
-    mpi_tutorial_repo_dest: "{{ mpi_data }}/mpitutorial"
-  tasks:
-    - include: test-mpi.yml
+- include: master.yml
+- include: slaves.yml
+- include: test_cluster.yml

provisioning/slaves.yml

@@ -0,0 +1,11 @@
+---
+
+- hosts: all
+  become: yes
+  tasks:
+    - include: tasks/create_hosts_file.yml
+
+- hosts: slaves
+  become: yes
+  roles:
+     - mpi_slave

provisioning/hosts.yml → provisioning/tasks/create_hosts_file.yml

@@ -1,6 +1,6 @@
 ---
 
-- name: Build hosts file
+- name: All hosts are contained in hosts file
   lineinfile:
     dest: /etc/hosts
     regexp: '.*{{ item }}$'

provisioning/test_cluster.yml

@@ -0,0 +1,45 @@
+---
+
+# Procedural process for testing that MPI is working across the cluster. Should perhaps be a script.
+
+- hosts: master
+  become: yes
+  vars:
+    - mpi_tutorial_zip: 'https://github.com/wesleykendall/mpitutorial/archive/master.zip'
+    - mpi_tutorial_repo_dest: "{{ mpi_data_share }}/mpitutorial-master"
+  tasks:
+    - name: Ensure Unzip present
+      apt:
+        name: unzip
+        state: present
+
+    - name: Extract MPI Tutorial project
+      unarchive:
+        src: 'https://github.com/wesleykendall/mpitutorial/archive/master.zip'
+        dest: "{{ mpi_data_share }}"
+        remote_src: yes
+
+    - name: Make Hello World project
+      make:
+        chdir: "{{ mpi_tutorial_repo_dest }}/tutorials/mpi-hello-world/code"
+
+    - name: Run Hello World program
+      shell: "su mpiuser -c 'mpirun --hostfile {{ mpi_hosts_file }} mpi_hello_world'"
+      args:
+        chdir: "{{ mpi_tutorial_repo_dest }}/tutorials/mpi-hello-world/code"
+      register: hello_world_result
+
+    - name: Output result of Hello World program
+      debug:
+        msg: "{{ hello_world_result.stdout_lines }}"
+
+    - name: Processor ran on all slaves
+      assert:
+        that: "hello_world_result.stdout.find('Hello world from processor {{ item }}') != -1"
+        msg: "{{ item }} failed to run the processor. Increase verbosity to more about the error."
+      with_items: "{{ groups['slaves'] }}"
+
+    - name: Clean up after test
+      file:
+        path: "{{ mpi_tutorial_repo_dest }}"
+        state: absent