1.0.0-beta.29

accelpy/__init__.py

@@ -16,7 +16,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-__version__ = '1.0.0-beta.28'
+__version__ = '1.0.0-beta.29'
 __copyright__ = "Copyright 2019 Accelize"
 __licence__ = "Apache 2.0"
 

accelpy/_ansible/__init__.py

@@ -35,6 +35,10 @@ def create_configuration(self, provider=None, application_type=None,
         """
         roles_local = dict()
         yaml_files = dict()
+        roles_versions = {}
+        application_roles = []
+        galaxy_roles = set()
+        roles_to_init = set()
 
         # Get sources
         for source_dir in get_sources_dirs(dirname(__file__), user_config):
@@ -55,6 +59,22 @@ def create_configuration(self, provider=None, application_type=None,
                     elif splitext(entry.name)[1] == '.yml':
                         yaml_files[entry.name] = entry.path
 
+        # Special case of tha application type is one or more Ansible role
+        if application_type == 'ansible_role':
+            application_type = ''
+            for package in variables['app_packages']:
+                role = package['name']
+                application_roles.append(role)
+
+                # Local role
+                if role in roles_local:
+                    roles_to_init.add(role)
+
+                # Role from galaxy
+                else:
+                    galaxy_roles.add(role)
+                    roles_versions[role] = package.get('version')
+
         # Filter roles
         roles = {name: path for name, path in roles_local.items()
                  if name.split('.', 1)[0] in get_sources_filters(
@@ -63,8 +83,7 @@ def create_configuration(self, provider=None, application_type=None,
         # Initialize roles
         role_dir = join(self._config_dir, 'roles')
         makedirs(role_dir, exist_ok=True)
-        galaxy_roles = set()
-        roles_to_init = set(roles)
+        roles_to_init.update(roles)
         initialized_roles = set()
 
         while roles_to_init:
@@ -89,6 +108,7 @@ def create_configuration(self, provider=None, application_type=None,
                 try:
                     # Formatted as "- role: name"
                     dep = dep_entry['role']
+                    roles_versions[dep] = dep_entry.get('version')
                 except TypeError:  # pragma: no cover
                     # May also be Formatted as "- name"
                     dep = dep_entry
@@ -99,7 +119,8 @@ def create_configuration(self, provider=None, application_type=None,
 
                 # Ansible Galaxy dependencies: To download
                 elif dep not in roles_local:
-                    galaxy_roles.add(dep)
+                    version = roles_versions.get(dep)
+                    galaxy_roles.add(f'{dep},{version}' if version else dep)
 
         # Install dependencies from Ansible Galaxy
         self.galaxy_install(galaxy_roles, roles_path=role_dir)
@@ -112,7 +133,8 @@ def create_configuration(self, provider=None, application_type=None,
         roles = sorted(roles)
         playbook[0]['roles'] = (
             [role for role in roles if role.endswith('.init')] +
-            [role for role in roles if not role.endswith('.init')])
+            [role for role in roles if not role.endswith('.init')] +
+            application_roles)
 
         yaml_write(playbook, join(self._config_dir, 'playbook.yml'))
 

accelpy/_ansible/common.user_override.yml

@@ -6,11 +6,11 @@
 
 # Enable OS security hardening
 # See "https://galaxy.ansible.com/dev-sec/os-hardening" for more information
-os_hardening: false
+os_hardening: true
 
 # Enable SSH security hardening
 # See "https://galaxy.ansible.com/dev-sec/ssh-hardening" for more information
-ssh_hardening: false
+ssh_hardening: true
 
 # Enable OS packages updates
 os_packages_updates: true

accelpy/_ansible/roles/common.init/meta/main.yml

@@ -8,11 +8,13 @@ dependencies:
     vars:
       ufw_manage_defaults: false
       os_auth_pam_passwdqc_enable: false
+      os_security_packages_clean: false  # To avoid remove "Vault" repositories
     when: os_hardening | bool
 
   - role: dev-sec.ssh-hardening
     vars:
       ssh_allow_users: "{{ ansible_user }}"
+      sftp_enabled: true
     when: ssh_hardening | bool
 
   # FPGA drivers

accelpy/_ansible/roles/container_service/molecule/default/app.Dockerfile

@@ -1,7 +1,6 @@
 FROM accelize/base:centos_7-aws_f1
 
-RUN yum install -y epel-release && \
-yum install -y python36 && \
+RUN yum install -y python36 && \
 rm -rf /var/cache/yum/*
 
 EXPOSE 8080

accelpy/_application.py

@@ -24,7 +24,7 @@
                   r'(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$'),
         'type': dict(
             required=True,
-            values=('container_service', 'kubernetes_node'),
+            values=('container_service', 'kubernetes_node', 'ansible_role'),
             default='container_service'),
         'variables': dict(
             value_type=dict,
@@ -34,7 +34,8 @@
         '_node': list,
         'type': dict(
             default='container_image',
-            values=('container_image', 'vm_image', 'kubernetes_yaml')),
+            values=('container_image', 'vm_image', 'kubernetes_yaml',
+                    'ansible_role')),
         'name': dict(
             required=True,),
         'version': dict(),
@@ -58,8 +59,8 @@
     'fpga': {
         '_node': dict,
         'image': dict(
-            required=True,
-            value_type=(list, str)),
+            value_type=(list, str),
+            default=[]),
         'driver': dict(
             values=('aws_f1', 'xilinx_xrt')),
         'driver_version': dict(),

accelpy/_packer/common.json

@@ -24,7 +24,9 @@
         "--extra-vars",
         "ansible_python_interpreter=auto",
         "--extra-vars",
-        "provider_required_driver={{user `provider_required_driver`}}"
+        "provider_required_driver={{user `provider_required_driver`}}",
+        "--extra-vars",
+        "master_node=false"
       ]
     }
   ],

container_base/centos_7-aws_f1.Dockerfile

@@ -27,3 +27,9 @@ rm -rf /tmp/* && \
 rm -rf /var/cache/yum/* && \
 groupadd -g 1001 fpgauser && \
 useradd -mN -u 1001 -g fpgauser appuser
+
+# Setup Xilinx XRT environment (Same as "source /opt/xilinx/xrt/setup.sh")
+ENV XILINX_XRT=/opt/xilinx/xrt
+ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/xilinx/xrt/bin
+ENV LD_LIBRARY_PATH=/opt/xilinx/xrt/lib
+ENV PYTHONPATH=/opt/xilinx/xrt/python

docs/application_ansible_role.rst

@@ -0,0 +1,39 @@
+Ansible role
+============
+
+The setup of the application is performed by one or more Ansible role.
+
+Si allow to fully configure the host as wish.
+
+Application definition
+----------------------
+
+This application type support the following package types:
+
+* `ansible_role`: A role available on Ansible Galaxy or in a local subdirectory
+  named `roles`.
+* `vm_image`: An image of an already provisioned virtual machine.
+
+Example snippet of application definition file:
+
+.. code-block:: yaml
+
+   application:
+     name: my_application
+     version: 1.0.2
+     type: ansible_role
+
+   package:
+     - name: ansible_galaxy_namespace.role_name
+       type: ansible_role
+     - name: ansible_galaxy_namespace.role_name
+       type: ansible_role
+
+.. note:: Any application definition published using `accely push` require that
+          roles are available on `Ansible Galaxy <https://galaxy.ansible.com>`_.
+
+Variables
+~~~~~~~~~
+
+This application support any variables. All variables will be available in
+Ansible roles.

docs/application_definition.rst

@@ -34,7 +34,7 @@ This section is a mapping of following key, values pairs:
   to filter configuration and use correct Ansible roles to deploy the
   application. For more information on each application type, see the
   "application type" section of the right menu. Predefined application types
-  are: `container_service`, `kubernetes_node`.
+  are: `ansible_role`, `container_service`, `kubernetes_node`.
 * `variables` (mapping of strings): Application type specific variables. See the
   application type documentation for more information.
 
@@ -60,6 +60,8 @@ This section is a list of mappings with following key values pairs:
 * `type` (string): **Required**. Package type. Each application type support a
      limited subset of package types. Predefined package types are:
 
+    * `ansible_role`: One or more Ansible role, available locally in a `roles`
+      subdirectory or on `Ansible Galaxy <https://galaxy.ansible.com>`_.
     * `container_image`: A Docker or OCI container image.
     * `kubernetes_yaml`: URL to a Kubernetes deployment, pod, service, ...
       YAML or JSON file.
@@ -151,7 +153,7 @@ device(s).
 * `driver` (string): The FPGA driver to use. If not specified, default to the
   Linux Kernel driver or the provider specific driver.
   Possible values : `aws_f1` (AWS F1 instances only), `xilinx_xrt` (Xilinx XRT).
-* `image` (string or list of string): **Required**. The FPGA bitstream image to
+* `image` (string or list of string): The FPGA bitstream image to
   use to program the FPGA. Depending the provider this can be an ID, a path or
   an URL. If multiple FPGA are required, must be a list of FPGA bitstream (One
   for each FPGA slot).

docs/application_kubernetes_node.rst

@@ -49,9 +49,10 @@ Variables
 
 This application support following variables:
 
-* `master_node`: If set to `true` (Default value), install the application as a
+* `master_node`: If set to `true`, install the application as a
   single master/node. If set to `false`, install the application as a node that
-  must be integrated with an existing Kubernetes infrastructure.
+  must be integrated with an existing Kubernetes infrastructure. Default to
+  `true` with `accelpy apply` and false with `accelpy build`.
 * `kubernetes_join_command`: Only if `master_node` is `false`. A command
   to run on the node to joint the Kubernetes master. This command may be
   generated by running `kubeadm token create --print-join-command` on the
@@ -150,4 +151,5 @@ Known issues
 
 In the single master/node mode, virtual machines image generated from
 `accelpy build` will not work properly once deployed due to the change of the
-IP address.
+IP address. For this reason, `master_node` is forced to `false` with
+`accelpy build`.

docs/example_kubeless_function.rst

@@ -182,22 +182,23 @@ Create a node image with Accelpy
 Since our application is now ready and fully working in master/node, time is
 ready to push it in production on our existing Kubernetes infrastructure on AWS.
 
-First, we need to pass some variables to Ansible to move to the
-single node mode and optionally specify the join command that will automatically
-join the node to an existing Kubernetes infrastructure. This can be done with
-the user override file.
+It is possible to pass some variables to Ansible to specify the join
+command that will automatically join the node to an existing Kubernetes
+infrastructure. This can be done with the user override file:
 
 .. code-block:: yaml
     :caption: ~/.accelize/common.user_override.yml
 
     ---
-    # Set as node only
-    master_node: false
-
     # Add join command from your master, you can get it with kubeadm
     # command: "kubeadm token create --print-join-command"
     kubernetes_join_command: join command
 
+.. note:: Using the override file is not the only way to execute the join
+          command on the node instance. It is also possible to execute it on
+          node instantiation (By example with the "cloud init" / "user data"
+          provided by some cloud providers).
+
 Then we run accelpy to build an image of the node and add its AMI to the
 application definition.
 

docs/getting_started.rst

@@ -158,6 +158,9 @@ Always using the same host image to generate new hosts ensure immutability, but
 don't forget to regularly regenerate the image and host that use it to ensure
 system software are up to date and keep them secure.
 
+.. warning:: Never share your image with untrusted people. It contain a copy
+             of your Accelize credential.
+
 SSH connection
 ~~~~~~~~~~~~~~
 

docs/index.rst

@@ -92,6 +92,7 @@ Documentation
    :maxdepth: 2
    :caption: Application types
 
+   application_ansible_role
    application_container_service
    application_kubernetes_node
 

examples/secureic_trng/application.yml

@@ -0,0 +1,29 @@
+---
+application:
+  product_id: secureic-trng
+  type: ansible_role
+  version: 1.0.0
+
+firewall_rules:
+  - start_port: 80
+    end_port: 80
+    protocol: tcp
+    direction: ingress
+  - start_port: 443
+    end_port: 443
+    protocol: tcp
+    direction: ingress
+
+fpga:
+  count: 1
+
+package:
+  type: ansible_role
+  name: secureic_trng
+
+accelize_drm:
+  conf:
+    licensing:
+      url: https://master.metering.accelize.com
+    drm:
+      frequency_mhz: 250

examples/secureic_trng/aws.user_override.tf

@@ -0,0 +1,25 @@
+/* Amazon Web Service user configuration */
+
+# Instance profile IAM role policy
+# ==================================
+#
+# Allow logging on Cloudwatch
+
+locals {
+  policy = <<EOF
+{
+  "Version": "2012-10-17", "Statement": [
+    {"Sid": "AllowDescribeFpgaImages",
+     "Effect": "Allow",
+     "Action": ["ec2:DescribeFpgaImages"],
+     "Resource": ["*"]},
+
+    {"Sid": "AllowCloudwatchLogging",
+     "Effect": "Allow",
+     "Action": ["logs:CreateLogGroup", "logs:CreateLogStream",
+                "logs:PutLogEvents", "logs:DescribeLogStreams"],
+     "Resource": ["*"]}
+  ]
+}
+EOF
+}

examples/secureic_trng/roles/secureic_trng/defaults/main.yml

@@ -0,0 +1,22 @@
+---
+# Set to non empty string to enable debug output
+debug_mode: ''
+
+# FPGA image binary
+fpga_binary: /usr/lib/python3.6/site-packages/trng/app.awsxclbin
+
+# Same as "accelize.accelize_drm" role varibles
+accelize_drm_driver_name: aws_f1
+accelize_drm_conf_dst: /etc/accelize_drm/conf.json
+accelize_drm_cred_dst: /root/.accelize_drm/cred.json
+
+# Nginx SSL certificate
+nginx_ssl_certificate:
+nginx_ssl_certificate_dst: /etc/pki/tls/certs/trng.crt
+nginx_ssl_trusted_certificate:
+nginx_ssl_trusted_certificate_dst: /etc/pki/tls/certs/trng_ca.crt
+nginx_ssl_certificate_key:
+nginx_ssl_certificate_key_dst: /etc/pki/tls/private/trng.key
+
+# Gunicorn socket path
+gunicorn_socket: /var/run/gunicorn.sock