From 997aec98e0e410c0f270db085625def8a0cb8f2c Mon Sep 17 00:00:00 2001 From: jgoutin Date: Thu, 11 Jul 2019 10:21:36 +0200 Subject: [PATCH] 1.0.0-beta.2 --- accelpy/__init__.py | 9 ++- accelpy/_ansible/__init__.py | 50 ++++++++++++--- .../molecule/default/playbook.yml | 4 +- .../roles/container_service/tasks/main.yml | 4 +- accelpy/_host.py | 64 ++++++++++++++----- accelpy/_terraform/aws.tf | 1 + docs/application_container_service.rst | 10 +++ docs/configuration.rst | 4 +- docs/getting_started.rst | 22 ++----- docs/index.rst | 33 +++++----- setup.py | 3 +- tests/test_container_service.yml | 4 +- 12 files changed, 141 insertions(+), 67 deletions(-) diff --git a/accelpy/__init__.py b/accelpy/__init__.py index 94bacbd..3a6cafc 100644 --- a/accelpy/__init__.py +++ b/accelpy/__init__.py @@ -16,10 +16,17 @@ See the License for the specific language governing permissions and limitations under the License. """ -__version__ = '1.0.0-beta.1' +__version__ = '1.0.0-beta.2' __copyright__ = "Copyright 2018 Accelize" __licence__ = "Apache 2.0" +from sys import version_info as _py +if (_py[0] < 3) or (_py[0] == 3 and _py[1] < 6): + from sys import version + raise ImportError( + 'Accelpy require Python 3.6 or more (Currently %s)' % version) +del _py + from accelpy._application import lint from accelpy._host import Host, iter_hosts diff --git a/accelpy/_ansible/__init__.py b/accelpy/_ansible/__init__.py index 29b4eaa..8277f8c 100644 --- a/accelpy/_ansible/__init__.py +++ b/accelpy/_ansible/__init__.py @@ -1,8 +1,8 @@ # coding=utf-8 """Ansible configuration""" from os import makedirs, fsdecode, scandir, listdir -from os.path import join, realpath, dirname -from sys import prefix, executable +from os.path import join, realpath, dirname, splitext, basename +from sys import executable from accelpy._common import ( yaml_read, yaml_write, call, get_sources_dirs, symlink, get_sources_filters) @@ -19,7 +19,7 @@ class Ansible: variables (dict): Ansible playbook variables. user_config (path-like object): User configuration directory. """ - _EXECUTABLE = join(realpath(prefix), 'bin/ansible') + _ANSIBLE_EXECUTABLE = None def __init__(self, config_dir, provider=None, application_type=None, variables=None, @@ -118,6 +118,40 @@ def create_configuration(self): yaml_write(playbook, self._playbook) + @classmethod + def _executable(cls): + """ + Find and return Ansible executable path from this Python environment. + + This ensure to execute a compatible version with the expected Python + version. + + returns: + str: path + """ + if cls._ANSIBLE_EXECUTABLE is None: + from ansible import __path__ + site_packages = dirname(__path__[0]) + record_path = '' + + with scandir(site_packages) as entries: + for entry in entries: + name = entry.name + if name.startswith( + 'ansible-') and splitext(name)[1] == '.dist-info': + record_path = join(entry.path, 'RECORD') + break + + with open(record_path, 'rt') as record: + for line in record: + path = line.split(',', 1)[0] + if basename(path) in ('ansible', 'ansible.exe'): + break + + cls._ANSIBLE_EXECUTABLE = realpath(join(site_packages, path)) + + return cls._ANSIBLE_EXECUTABLE + def _ansible(self, *args, utility=None, check=True, pipe_stdout=False, **run_kwargs): """ @@ -134,10 +168,10 @@ def _ansible(self, *args, utility=None, check=True, pipe_stdout=False, Returns: subprocess.CompletedProcess: Ansible call result. """ - return call([executable, f"{self._EXECUTABLE}-{utility}" if utility else - self._EXECUTABLE] + list(args), - cwd=self._config_dir, check=check, pipe_stdout=pipe_stdout, - **run_kwargs) + return call( + [executable, f"{self._executable()}-{utility}" if utility else + self._executable] + list(args), cwd=self._config_dir, check=check, + pipe_stdout=pipe_stdout, **run_kwargs) def lint(self): """ @@ -167,4 +201,4 @@ def playbook_exec(cls): Returns: str: command """ - return f'{cls._EXECUTABLE}-playbook' + return f'{cls._executable()}-playbook' diff --git a/accelpy/_ansible/roles/container_service/molecule/default/playbook.yml b/accelpy/_ansible/roles/container_service/molecule/default/playbook.yml index 126b6e6..8479730 100644 --- a/accelpy/_ansible/roles/container_service/molecule/default/playbook.yml +++ b/accelpy/_ansible/roles/container_service/molecule/default/playbook.yml @@ -4,8 +4,8 @@ roles: - role: container_service vars: - package_name: accelize/base - package_version: test + package_name: accelize/accelpy-ci + package_version: container_service firewall_rules: - start_port: 8080 end_port: 8080 diff --git a/accelpy/_ansible/roles/container_service/tasks/main.yml b/accelpy/_ansible/roles/container_service/tasks/main.yml index a0961e3..7625904 100644 --- a/accelpy/_ansible/roles/container_service/tasks/main.yml +++ b/accelpy/_ansible/roles/container_service/tasks/main.yml @@ -15,13 +15,15 @@ apt_repository: repo: ppa:projectatomic/ppa when: rootless|bool + retries: 10 + delay: 1 - name: Ensure Podman is installed apt: name: podman state: present update_cache: true - retries: 3 + retries: 10 delay: 1 when: rootless|bool diff --git a/accelpy/_host.py b/accelpy/_host.py index d6addb2..db0b02e 100644 --- a/accelpy/_host.py +++ b/accelpy/_host.py @@ -1,9 +1,9 @@ """Manage hosts life-cycle""" from os import chmod, fsdecode, makedirs, scandir, symlink -from os.path import isabs, isdir, join, realpath +from os.path import isabs, isdir, isfile, join, realpath from accelpy._application import Application -from accelpy._common import HOME_DIR, json_read, json_write +from accelpy._common import HOME_DIR, json_read, json_write, get_sources_dirs from accelpy.exceptions import ConfigurationException CONFIG_DIR = join(HOME_DIR, 'hosts') @@ -70,10 +70,15 @@ def __init__(self, name=None, application=None, provider=None, self._output_json = join(self._config_dir, 'output.json') self._accelize_drm_conf_json = join( self._config_dir, 'accelize_drm_conf.json') + self._accelize_drm_cred_json = join(self._config_dir, 'cred.json') # Create a new configuration config_exists = isdir(self._config_dir) if not config_exists and application: + + # Ensure config is cleaned on creation error + self._keep_config = False + # Create target configuration directory and remove access to other # users since Terraform state files may content sensible data and # directory may contain SSH private key @@ -91,6 +96,11 @@ def __init__(self, name=None, application=None, provider=None, # Get application and add it as link with configuration self._application_yaml = realpath(fsdecode(application)) + + # Check Accelize Requirements + self._init_accelize_drm() + + # Add link to configuration symlink(self._application_yaml, join( self._config_dir, 'application.yml')) @@ -99,6 +109,8 @@ def __init__(self, name=None, application=None, provider=None, self._ansible.create_configuration() self._packer.create_configuration() + self._keep_config = keep_config + # Load an existing configuration elif config_exists: @@ -117,6 +129,36 @@ def __init__(self, name=None, application=None, provider=None, 'Require at least an existing host name, or an ' 'application to create a new host.') + def _init_accelize_drm(self): + """Initialize Accelize DRM requirements""" + + # Create configuration file from application + accelize_drm_enable = self._app('accelize_drm', 'use_service') + accelize_drm_conf = self._app('accelize_drm', 'conf') + + if accelize_drm_enable and not accelize_drm_conf: + raise ConfigurationException( + 'Application definition section "accelize_drm" require ' + '"conf" value to be specified if "use_service" is ' + 'specified.') + + json_write(accelize_drm_conf, self._accelize_drm_conf_json) + + # Get credentials file from user configuration + for src in get_sources_dirs(self._user_config): + + cred_path = join(src, 'cred.json') + + if isfile(cred_path): + symlink(cred_path, self._accelize_drm_cred_json) + break + else: + raise ConfigurationException( + 'No Accelize DRM credential found. Please, make sure to ' + f'have your "cred.json" file installed in "{HOME_DIR}", ' + f'current directory or path specified with the ' + f'"user_config" argument.') + def __enter__(self): return self @@ -296,19 +338,6 @@ def _ansible(self): # Lazy import: May not be used all time from accelpy._ansible import Ansible - # Get Accelize DRM configuration - accelize_drm_enable = self._app('accelize_drm', 'use_service') - accelize_drm_conf = self._app('accelize_drm', 'conf') - - if accelize_drm_enable and not accelize_drm_conf: - # Check configuration presence instead of wait role failure - raise ConfigurationException( - 'Application definition section "accelize_drm" require ' - '"conf" value to be specified if "use_service" is ' - 'specified.') - - json_write(accelize_drm_conf, self._accelize_drm_conf_json) - # Set Ansible variables variables = dict( fpga_image=self._app('fpga', 'image'), @@ -320,9 +349,10 @@ def _ansible(self): package_name=self._app('package', 'name'), package_version=self._app('package', 'version'), package_repository=self._app('package', 'repository'), - accelize_drm_disabled=not accelize_drm_enable, + accelize_drm_disabled=not self._app('accelize_drm', + 'use_service'), accelize_drm_conf_src=self._accelize_drm_conf_json, - accelize_drm_cred_src=join(self._config_dir, 'cred.json') + accelize_drm_cred_src=self._accelize_drm_cred_json ) self._ansible_config = Ansible( diff --git a/accelpy/_terraform/aws.tf b/accelpy/_terraform/aws.tf index dc55e7d..96523b6 100644 --- a/accelpy/_terraform/aws.tf +++ b/accelpy/_terraform/aws.tf @@ -187,6 +187,7 @@ resource "aws_spot_instance_request" "instance_spot" { wait_for_fulfillment = true provisioner "local-exec" { # "tags" apply to spot instance request and needs to be applied to instance + # https://github.com/terraform-providers/terraform-provider-aws/issues/32 command = "aws ec2 create-tags --resources ${self.spot_instance_id} --tags Key=Name,Value=${local.name}" } diff --git a/docs/application_container_service.rst b/docs/application_container_service.rst index 88c5aeb..2660e9c 100644 --- a/docs/application_container_service.rst +++ b/docs/application_container_service.rst @@ -6,6 +6,16 @@ The application is a container infinitely running in background. The application manager is a systemd service that run the container once on boot. +Prerequisites +------------- + +To create this kind of application, you need to create a Docker container image +and push it on a registry like Docker-Hub. Following links from Docker +documentation can help to start with container images: + +* `Develop with Docker `_ +* `Repositories `_ + Container configuration ----------------------- diff --git a/docs/configuration.rst b/docs/configuration.rst index c0bbb50..a68d6ff 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -51,8 +51,8 @@ Filenames are also filtered using their names, only files that match following pasterns are imported to the configuration. With the filter name equal to `common`, the provider name or the application -type. Files that starts with the filter name after being split by `.` are -kept in the configuration. +type. Files that starts with the filter name after being split by `.` are kept +in the configuration. Tool specific configuration handling ------------------------------------ diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 0409daa..0cb964b 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -21,10 +21,9 @@ The installation is performed using Pip: Ansible and all required Python packages are installed automatically by Pip. -HashiCorp utilities (Terraform & Packer) are managed automatically by -accelpy. It ensures that the version used is up to date, -downloads and installs the tool if necessary after checking its signature and -integrity. +HashiCorp utilities (Terraform & Packer) are managed automatically by accelpy. +It ensures that the version used is up to date, downloads and installs the tool +if necessary after checking its signature and integrity. Application definition ---------------------- @@ -155,24 +154,13 @@ system software are up to date and keep them secure. SSH connection ~~~~~~~~~~~~~~ -It is possible to connect application host using SSH. To do this the utility -allow user to retrieve all required information and can add/remove the required -SSH key to the SSH agent. +It is possible to connect application host using SSH using information returned +by the utility. Example with OpenSSH: .. code-block:: bash - # Add the SSH key to the SSH agent - accelpy ssh_agent_add - - # Connect to SSH using host information - ssh -Yt $(accelpy ssh_user)@$(accelpy public_ip) - - # Remove the SSH key from the agent - accelpy ssh_agent_remove - - # It is also possible to connect without using the SSH agent ssh -Yt -i $(accelpy ssh_private_key) $(accelpy ssh_user)@$(accelpy public_ip) .. note:: By default, the utility generate a new SSH key for each configuration, diff --git a/docs/index.rst b/docs/index.rst index 07d9006..4b3df8b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,13 +29,13 @@ Provision in the multi-Cloud `Packer support `_. Provision an immutable infrastructure - Once the application is ready, *accelpy* allow creating an image - of it for all required provider and then use this image to deploy an - immutable infrastructure. + Once the application is ready, *accelpy* allow creating an image of it for + all required provider and then use this image to deploy an immutable + infrastructure. Don't care about the FPGA requirements - *accelpy* configure the host with required FPGA drivers and ensure - the FPGA bitstream is loaded before starting the application. + *accelpy* configure the host with required FPGA drivers and ensure the FPGA + bitstream is loaded before starting the application. Configure and scale as you need *accelpy* provides default infrastructure configuration that can be @@ -45,30 +45,31 @@ Configure and scale as you need This allows generating the scalable infrastructure that fit your needs. Protect your FPGA application - *accelpy* is integrated into the Accelize solution and provides all - the tools to provision application protected by the Accelize DRM. + *accelpy* is integrated into the + `Accelize solution `_ and provides all + the tools to provision application protected by the + `Accelize DRM `_. Containerize your application - *accelpy* allow to package the software part of your application as - a *Docker* container image and a single YAML configuration file. + *accelpy* allow to package the software part of your application as a + *Docker* container image and a single YAML configuration file. Run without configuration *accelpy* provides default ready to use configuration to immediately provision a single host infrastructure on a subset of providers. Use and integrate it easily - *accelpy* can be operated using the command line interface - interface or the Python API. + *accelpy* can be operated using the command line interface or the Python + API. Deploy application on secure host that follow DevOps good practices - The default configurations provided with *accelpy* are done with - DevOps good practices and enhanced security in mind. Relevant + The default configurations provided with *accelpy* are done with DevOps + good practices and enhanced security in mind. Relevant `DevSec hardening baselines `_ are applied by default. Be free to use it as you want - *accelpy* generate a configuration for your application that can - then be used without the utility itself as part of your own provisioning - project. + *accelpy* generate a configuration for your application that can then be + used without the utility itself as part of your own provisioning project. The entire project is open source and based on open source tools. diff --git a/setup.py b/setup.py index e36a8ad..0f6912b 100755 --- a/setup.py +++ b/setup.py @@ -44,7 +44,8 @@ python_requires='>=3.6', install_requires=[ 'requests>=2.20.0', - 'ansible>=2.8' + 'ansible>=2.8', + 'awscli>=1.16' # To remove once Terraform support spot instance tagging ], setup_requires=['setuptools'], tests_require=['pytest', 'molecule[docker]'], diff --git a/tests/test_container_service.yml b/tests/test_container_service.yml index 041e9a6..7885e46 100644 --- a/tests/test_container_service.yml +++ b/tests/test_container_service.yml @@ -16,8 +16,8 @@ fpga: count: 1 package: - name: accelize/base - version: test + name: accelize/accelpy-ci + version: container_service type: container_image accelize_drm: