1.0.0-beta.6

MANIFEST.in

@@ -1,3 +1,3 @@
 recursive-include accelpy *
-recursive-exclude accelpy *.pyc
+recursive-exclude accelpy *.pyc .ansible-lint
 prune accelpy/_ansible/roles/*/molecule

accelpy/__init__.py

@@ -16,7 +16,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-__version__ = '1.0.0-beta.5'
+__version__ = '1.0.0-beta.6'
 __copyright__ = "Copyright 2018 Accelize"
 __licence__ = "Apache 2.0"
 

accelpy/__main__.py

@@ -321,10 +321,13 @@ def _run_command():
         parser.exit()
 
     except (AccelizeException, OSError) as exception:
-        parser.error(str(exception))
+        from accelpy._common import debug
+        if not debug():
+            parser.error(str(exception))
+        raise
 
     except KeyboardInterrupt:
-        parser.error("Interrupted by user")
+        parser.exit(status=1, message="Interrupted by user\n")
 
 
 if __name__ == '__main__':

accelpy/_ansible/playbook.yml

@@ -3,31 +3,6 @@
   become: true
   pre_tasks:
 
-    - name: Ensure Apt cache is up to date
-      apt:
-        update_cache: true
-      retries: 10
-      delay: 1
-      when: ansible_os_family == 'Debian'
-
-    - name: Ensure minimal Apt dependencies are installed
-      apt:
-        name:
-          - apt-transport-https
-          - gnupg
-          - software-properties-common
-        state: present
-        install_recommends: false
-        update_cache: true
-      retries: 10
-      delay: 1
-      when: ansible_os_family == 'Debian'
-
-    - name: Ensure minimal Yum dependencies are installed
-      yum:
-        name: gnupg
-        state: present
-        install_weak_deps: false
-      retries: 10
-      delay: 1
-      when: ansible_os_family == 'RedHat'
+    # Set some variables if defined
+    - set_fact: xilinx_xrt_version="{{ fpga_driver_version }}"
+      when: fpga_driver_version is defined

accelpy/_ansible/roles/common.init/.ansible-lint

@@ -0,0 +1,5 @@
+skip_list:
+  - '701'
+  - '702'
+  - '703'
+  - '704'

accelpy/_ansible/roles/common.init/meta/main.yml

@@ -1,27 +1,33 @@
 ---
-galaxy_info:
-  author: Accelize
-  description: 'Initialize host with FPGA driver and common tasks.'
-  min_ansible_version: 2.8
-  license: Apache License 2.0
-  platforms:
-    - name: Ubuntu
-      versions:
-        - bionic
-
 dependencies:
+  # System update
+  - role: common.update_system
+
   # Security hardening
   - role: dev-sec.os-hardening
     vars:
       ufw_manage_defaults: false
       os_auth_pam_passwdqc_enable: false
+    when: os_hardening | bool
+
+  - role: dev-sec.ssh-hardening
+    vars:
+      ssh_allow_users: "{{ remote_user }}"
+    when: ssh_hardening | bool
+
+  # System reboot
+  - role: common.reboot_system
 
   # FPGA drivers
+
   - role: 'accelize.aws_fpga'
     vars:
       aws_fpga_allow_non_root: true
       aws_fpga_sdk_override_group: true
     when: accelize_drm_driver_name == 'aws_f1'
 
+  - role: 'accelize.xilinx_xrt'
+    when: accelize_drm_driver_name == 'xrt'
+
   # Accelize DRM
   - accelize.accelize_drm

accelpy/_ansible/roles/common.reboot_system/.ansible-lint

@@ -0,0 +1,5 @@
+skip_list:
+  - '701'
+  - '702'
+  - '703'
+  - '704'

accelpy/_ansible/roles/common.reboot_system/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Ensure system is rebooted
+  reboot:

accelpy/_ansible/roles/common.update_system/.ansible-lint

@@ -0,0 +1,5 @@
+skip_list:
+  - '701'
+  - '702'
+  - '703'
+  - '704'

accelpy/_ansible/roles/common.update_system/tasks/main.yml

@@ -0,0 +1,26 @@
+---
+- name: Ensure Apt cache is up to date
+  apt:
+    update_cache: true
+  retries: 10
+  delay: 1
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure system is up to date
+  apt:
+    name: "*"
+    state: latest
+    update_cache: true
+    autoremove: true
+    purge: true
+  retries: 10
+  delay: 1
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure system is up to date
+  yum:
+    name: "*"
+    state: latest
+  retries: 10
+  delay: 1
+  when: ansible_os_family == 'RedHat'

accelpy/_ansible/roles/container_service/.ansible-lint

@@ -0,0 +1,5 @@
+skip_list:
+  - '701'
+  - '702'
+  - '703'
+  - '704'

accelpy/_ansible/roles/container_service/meta/main.yml

@@ -1,14 +1,4 @@
 ---
-galaxy_info:
-  author: Accelize
-  description: 'Install an application as Container service.'
-  min_ansible_version: 2.8
-  license: Apache License 2.0
-  platforms:
-    - name: Ubuntu
-      versions:
-        - bionic
-
 dependencies:
   - role: geerlingguy.docker
     when: not rootless|bool

accelpy/_ansible/roles/kubernetes_node/.ansible-lint

@@ -0,0 +1,5 @@
+skip_list:
+  - '701'
+  - '702'
+  - '703'
+  - '704'

accelpy/_ansible/roles/kubernetes_node/meta/main.yml

@@ -1,14 +1,4 @@
 ---
-galaxy_info:
-  author: Accelize
-  description: 'Install a FPGA ready Kubernetes cluster.'
-  min_ansible_version: 2.8
-  license: Apache License 2.0
-  platforms:
-    - name: Ubuntu
-      versions:
-        - bionic
-
 dependencies:
   - role: geerlingguy.docker
     vars:

accelpy/_application.py

@@ -94,6 +94,13 @@
             value_type=dict,
             default={},
         ),
+    },
+    'test': {
+        '_node': dict,
+        'shell': dict(
+            desc='Shell command used to test application that should return a 0'
+                 ' code in cas of success.'
+        )
     }
 }
 
@@ -194,6 +201,12 @@ def _validate(self, definition):
             self._validate_section(
                 node_type, section, section_name, section_format)
 
+        # Check for unknown sections
+        for section_name in definition:
+            if section_name not in FORMAT:
+                raise ConfigurationException(
+                    f'Unknown "{section_name}" section.')
+
         return definition
 
     def _validate_section(
@@ -351,9 +364,16 @@ def _validate_env_node(self, node, node_format, section_name):
             self._environments.add(env)
             env_found = True
             env_node = node[env]
+
+            # Env that is not a dict is likely an unknown key
+            if not isinstance(env_node, dict) or env_node == '_node':
+                raise ConfigurationException(
+                    f'Unknown "{env_node}" key in "{section_name}" section.')
+
+            # Check environment integrity
             for key in node_format:
 
-                if key == '_node' or not isinstance(env_node, dict):
+                if key == '_node':
                     continue
 
                 value = env_node.get(key, node.get(key))
@@ -370,4 +390,11 @@ def _validate_env_node(self, node, node_format, section_name):
                     env_node[key] = Application._check_value(
                         key, key_format, value, section_name)
 
+            # Check for unknown keys in environment
+            for key in env_node:
+                if key not in node_format or key == '_node':
+                    raise ConfigurationException(
+                        f'Unknown "{key}" key in "{section_name}" section '
+                        f'for "{env}" environment.')
+
         return env_found

accelpy/_common.py

@@ -4,7 +4,8 @@
 from json import dump as _json_dump, load as _json_load
 from json.decoder import JSONDecodeError as _JSONDecodeError
 from os import (fsdecode as _fsdecode, symlink as _symlink, chmod as _chmod,
-                makedirs as _makesdirs, scandir as _scandir)
+                makedirs as _makesdirs, scandir as _scandir,
+                environ as _environ)
 from os.path import (
     expanduser as _expanduser, isdir as _isdir, realpath as _realpath,
     join as _join, dirname as _dirname, basename as _basename,
@@ -252,3 +253,23 @@ def get_python_package_entry_point(package, entry_point):
 
         if _isfile(entry_point_path):
             return entry_point_path
+
+
+def no_color():
+    """
+    If "ACCELPY_NO_COLOR" environment variable is set, return True.
+
+    Returns:
+        bool: True if no color mode.
+    """
+    return bool(_environ.get("ACCELPY_NO_COLOR", False))
+
+
+def debug():
+    """
+    If "ACCELPY_DEBUG" environment variable is set, return True.
+
+    Returns:
+        bool: True if debug mode.
+    """
+    return bool(_environ.get("ACCELPY_DEBUG", False))

accelpy/_hashicorp/__init__.py

@@ -295,7 +295,7 @@ def _exec(self, *args, check=True, pipe_stdout=False, **run_kwargs):
         Returns:
             subprocess.CompletedProcess: Utility call result.
         """
-        return call([self._get_executable()] + list(args),
+        return call([self._get_executable()] + [arg for arg in args if arg],
                     cwd=self._config_dir, check=check, pipe_stdout=pipe_stdout,
                     **run_kwargs)
 

accelpy/_host.py

@@ -3,7 +3,8 @@
 from os.path import isabs, isdir, isfile, join, realpath
 
 from accelpy._application import Application
-from accelpy._common import HOME_DIR, json_read, json_write, get_sources_dirs
+from accelpy._common import (
+    HOME_DIR, json_read, json_write, get_sources_dirs, no_color, debug)
 from accelpy.exceptions import ConfigurationException
 
 CONFIG_DIR = join(HOME_DIR, 'hosts')
@@ -373,12 +374,14 @@ def _packer(self):
         if not self._packer_config:
             # Lazy import: May not be used all time
             from accelpy._packer import Packer
+            from accelpy._ansible import Ansible
 
             variables = {
                 f'provider_param_{index}': value
                 for index, value in enumerate(
                     (self._provider or '').split(','))}
             variables['image_name'] = self._name
+            variables['ansible'] = Ansible.playbook_exec()
 
             self._packer_config = Packer(
                 provider=self._provider, config_dir=self._config_dir,
@@ -397,8 +400,22 @@ def _terraform(self):
         if not self._terraform_config:
             # Lazy import: May not be used all time
             from accelpy._terraform import Terraform
+            from accelpy._ansible import Ansible
+
+            no_color_mode = no_color()
+            debug_mode = debug()
+            ansible_env = ' '.join(f'{key}={value}' for key, value in {
+                'ANSIBLE_DISPLAY_SKIPPED_HOSTS': debug_mode,
+                'ANSIBLE_DISPLAY_OK_HOSTS': debug_mode,
+                'ANSIBLE_HOST_KEY_CHECKING': False,
+                'ANSIBLE_DEPRECATION_WARNINGS': debug_mode,
+                'ANSIBLE_ACTION_WARNINGS': debug_mode,
+                'ANSIBLE_FORCE_COLOR': not no_color_mode,
+                'ANSIBLE_NOCOLOR': no_color_mode
+            }.items())
 
             variables = dict(
+                ansible=f"{ansible_env} {Ansible.playbook_exec()}",
                 firewall_rules=self._application['firewall_rules'],
                 fpga_count=self._app('fpga', 'count'),
                 package_vm_image=self._app('package', 'name')
@@ -426,22 +443,34 @@ def _application(self):
 
         return self._application_definition
 
+    def _terraform_has_state(self):
+        """
+        Check if Terraform has a state.
+
+        Returns:
+            bool: True If Terraform has state.
+        """
+        try:
+            return bool(self._terraform.state_list())
+        except (ConfigurationException, FileNotFoundError):
+            return False
+
     def _clean_up(self):
         """
         Clean up configuration directory if there is no remaining resource
         within the Terraform state.
         """
         if self._config_dir is not None and isdir(self._config_dir):
             # Destroy managed infrastructure if exists
-            if self._destroy_on_exit and self._terraform.state_list():
+            if self._destroy_on_exit and self._terraform_has_state():
                 self._terraform.destroy(quiet=True)
 
             # Check if there is some remaining resources in state file
             # If it is the case, do not clean up configuration to allow
             # to reuse it
-            if not self._terraform.state_list() and not self._keep_config:
+            if not self._terraform_has_state() and not self._keep_config:
 
                 # Lazy import: Only used on remove
                 from shutil import rmtree
 
-                rmtree(self._config_dir)
+                rmtree(self._config_dir, ignore_errors=True)