1.0.0-beta.20

accelpy/__init__.py

@@ -16,7 +16,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-__version__ = '1.0.0-beta.19'
+__version__ = '1.0.0-beta.20'
 __copyright__ = "Copyright 2019 Accelize"
 __licence__ = "Apache 2.0"
 

accelpy/__main__.py

@@ -268,12 +268,11 @@ def _application_completer(prefix, parsed_args, **__):
 
     # If not 100% sure the application is a local file, get applications from
     # the web service, but avoid to call it every time for performance reason.
-    # - "." may be found in paths likes "./" or as file extension delimiter, it
-    #   cannot be found in the product_id, but may be found in the version
-    #   (In this case, the delimiter ":" is also present).
+    # - Only path should starts with "." or "/"
     # - Product ID is in format "vendor/library/name" should not contain more
     #   than 2 "/"
-    if ("." not in prefix or ':' in prefix) and prefix.count('/') <= 2:
+    if (not prefix.startswith('.') and not prefix.startswith('/') and
+            prefix.count('/') <= 2):
         from itertools import chain
         from accelpy._application import Application
         from accelpy.exceptions import (

accelpy/_ansible/roles/container_service/molecule/default/app_server.py

@@ -12,8 +12,9 @@ def do_GET(self):
         """GET"""
         process = run(['/opt/xilinx/xrt/bin/awssak', 'list'],
                       stderr=STDOUT, stdout=PIPE)
-        self.send_response(500 if (
-                process.returncode or b"[0] " not in process.stdout) else 200)
+        self.send_response(
+            500 if (process.returncode or b"error" in process.stdout.lower())
+            else 200)
         self.end_headers()
         self.wfile.write(process.stdout)
 

accelpy/_ansible/roles/kubernetes_node/defaults/main.yml

@@ -1,3 +1,4 @@
 ---
 single_node: true
 master_node: true
+kubernetes_join_command: pwd  # Do nothing, just to avoid crash

accelpy/_application.py

@@ -1,5 +1,6 @@
 # coding=utf-8
 """Application Definition"""
+from json import dumps
 from os import fsdecode
 from re import fullmatch
 
@@ -89,10 +90,12 @@ class Application:
         definition (path-like object or dict):
             Path to yaml definition file or dict of the content of the
             definition.
+        configuration_id (int): ID of configuration in Accelize web service.
     """
 
-    def __init__(self, definition):
+    def __init__(self, definition, configuration_id=None):
         self._providers = set()
+        self._configuration_id = configuration_id
 
         # Load from dict
         if isinstance(definition, dict):
@@ -125,37 +128,50 @@ def from_id(cls, application):
 
         Args:
             application (str): Application if format "product_id:version" or
-                "product_id".
+                "product_id". If version is not specified, last stable version
+                available will be used.
 
         Returns:
             Application: Application definition.
         """
-        # Get product ID and version
+        definition = cls._get_definition(application)
+        configuration_id = definition['application'].pop('configuration_id')
+        return cls(definition, configuration_id=configuration_id)
+
+    @staticmethod
+    def _get_definition(application):
+        """
+        Load application from Accelize web service.
+
+        Args:
+            application (str): Application if format "product_id:version" or
+                "product_id".
+
+        Returns:
+            dict: Raw definition from web service.
+        """
+        params = dict(limit=1)
         try:
-            product_id, version = application.split(':', 1)
+            params['product_id'], params['version'] = application.split(':', 1)
         except ValueError:
-            product_id = application
-            version = None
-
-        # Get definition from server
-        response = request.query('/auth/getapplicationdefinition/',
-                                 dict(product_id=product_id, version=version))
-        return cls(response)
+            params['product_id'] = application
+        return request.query('/auth/objects/productconfiguration/',
+                             params=params)['results'][0]
 
     @staticmethod
     def list(prefix=''):
         """
         List available applications on Accelize web service.
 
         Args:
-            product_id (str): Product ID linked to the application.
             prefix (str): Product ID prefix to filter.
 
         Returns:
             list of str: products.
         """
-        return request.query('/auth/listapplicationdefinitions/',
-                             dict(prefix=prefix))
+        return request.query(
+            '/auth/objects/productconfigurationlistproduct/', params=dict(
+                product_id__startswith=prefix) if prefix else None)['results']
 
     @staticmethod
     def list_versions(product_id, prefix=''):
@@ -169,15 +185,49 @@ def list_versions(product_id, prefix=''):
         Returns:
             list of str: versions.
         """
-        return request.query('/auth/listapplicationdefinitionversions/',
-                             dict(product_id=product_id, prefix=prefix))
+        params = dict(product_id=product_id)
+        if prefix:
+            params['version__startswith'] = prefix
+        return request.query('/auth/objects/productconfigurationlistversion/',
+                             params=params)['results']
 
     def push(self):
         """
         Push application definition on Accelize web service.
         """
-        return request.query('/auth/pushapplicationdefinition/',
-                             self._definition, 'post')
+        self._configuration_id = request.query(
+            '/auth/objects/productconfiguration/',
+            data=dumps(self._clean_definition),
+            method='post')['application']['configuration_id']
+
+    @classmethod
+    def delete(cls, application):
+        """
+        Delete application definition on Accelize web service.
+
+        Args:
+            application (str or int): Application if format "product_id:version"
+                or "product_id" or configuration ID in Accelize web service.
+                If specific version of ID not specified, will delete the last
+                stable version.
+        """
+        if not isinstance(application, int):
+            # Get configuration ID
+            application = cls._get_definition(
+                application)['application']['configuration_id']
+
+        request.query(f'/auth/objects/productconfiguration/{application}/',
+                      method='delete')
+
+    @property
+    def configuration_id(self):
+        """
+        Configuration ID in Accelize web service.
+
+        Returns:
+            int: ID
+        """
+        return self._configuration_id
 
     @property
     def providers(self):
@@ -249,7 +299,39 @@ def save(self, path=None):
         Args:
             path (path-like object): Path where save Yaml definition file.
         """
-        yaml_write(self._definition, path or self._path)
+        yaml_write(self._clean_definition, path or self._path)
+
+    @property
+    def _clean_definition(self):
+        """
+        Return definition cleaned up from empty values.
+
+        Returns:
+            dict: definition
+        """
+        from copy import deepcopy
+
+        definition = deepcopy(self._definition)
+        for section_name in tuple(definition):
+            section = definition[section_name]
+
+            if isinstance(section, list):
+                for element in tuple(section):
+                    for key in tuple(element):
+                        if not element[key] and element[key] is not False:
+                            del element[key]
+                    if not element:
+                        section.remove(element)
+
+            else:
+                for key in tuple(section):
+                    if not section[key] and section[key] is not False:
+                        del section[key]
+
+            if not section:
+                del definition[section_name]
+
+        return definition
 
     def _validate(self, definition):
         """

accelpy/_common.py

@@ -25,9 +25,6 @@
 HOME_DIR = _expanduser('~/.accelize')
 CACHE_DIR = _join(HOME_DIR, '.cache')
 
-#: Accelize endpoint
-ACCELIZE_ENDPOINT = 'https://master.metering.accelize.com'
-
 # Ensure directory exists and have restricted access rights
 _makesdirs(CACHE_DIR, exist_ok=True)
 _chmod(HOME_DIR, 0o700)
@@ -423,11 +420,14 @@ class _Request:
     Request to accelize server.
     """
     _TIMEOUT = 10
+    _RETRIES = 3
+    _ENDPOINT = 'https://master.metering.accelize.com'
 
     def __init__(self):
         self._token_expire = None
         self._token = None
         self._session = None
+        self._endpoint = self._ENDPOINT
 
     def _get_session(self):
         """
@@ -439,18 +439,28 @@ def _get_session(self):
         if self._session is None:
             # Lazy import, may never be called
             from requests import Session
+            from requests.adapters import HTTPAdapter
+            from urllib3.util.retry import Retry
+
+            # Create session with automatic retries on some error codes
+            adapter = HTTPAdapter(max_retries=Retry(
+                total=self._RETRIES, read=self._RETRIES, connect=self._RETRIES,
+                backoff_factor=0.3, status_forcelist=(408, 500, 502, 504)))
+
             self._session = Session()
+            self._session.mount('http://', adapter)
+            self._session.mount('https://', adapter)
 
         return self._session
 
-    def query(self, path, data=None, method='get'):
+    def query(self, path, method='get', **kwargs):
         """
         Performs a query.
 
         Args:
             path (str): URL path
-            data (dict): data.
             method (str): Request method.
+            kwargs: Requests query kwargs.
 
         Returns:
             dict or list: Response.
@@ -459,12 +469,13 @@ def query(self, path, data=None, method='get'):
 
         while True:
             # Get response
+            token = self._get_token()
             response = getattr(self._get_session(), method)(
-                ACCELIZE_ENDPOINT + path, data=data,
-                headers={"Authorization": "Bearer " + self._get_token(),
-                         "Content-Type": "application/json",
-                         "Accept": "application/vnd.accelize.v1+json"},
-                timeout=self._TIMEOUT)
+                self._endpoint + path, headers={
+                    "Authorization": "Bearer " + token,
+                    "Content-Type": "application/json",
+                    "Accept": "application/vnd.accelize.v1+json"},
+                timeout=self._TIMEOUT, **kwargs)
 
             # Token may be invalid
             if response.status_code == 401 and not retried:
@@ -476,7 +487,10 @@ def query(self, path, data=None, method='get'):
             elif response.status_code >= 300:
                 raise _ConfigurationException(self._get_error_message(response))
 
-            return response.json()
+            try:
+                return response.json()
+            except _JSONDecodeError:
+                return
 
     def _get_error_message(self, response):
         """
@@ -489,7 +503,7 @@ def _get_error_message(self, response):
             str: Error message.
         """
         try:
-            return response.json()["error"]
+            return response.json()["detail"]
         except (KeyError, _JSONDecodeError):
             return response.text
 
@@ -514,14 +528,17 @@ def _get_token(self):
             client_id = credentials['client_id']
             client_secret = credentials['client_secret']
 
+            # Endpoint override in credentials file
+            self._endpoint = credentials.get('endpoint', self._ENDPOINT)
+
             # Try to get from cache
             try:
                 self._token, self._token_expire = get_cli_cache(client_id)
 
             # Try to get from web service
             except TypeError:
                 response = self._get_session().post(
-                    f'{ACCELIZE_ENDPOINT}/o/token/',
+                    f'{self._endpoint}/o/token/',
                     data={"grant_type": "client_credentials"},
                     auth=(client_id, client_secret),
                     timeout=self._TIMEOUT)

buildspec.yml

@@ -12,10 +12,11 @@ phases:
       python: 3.7
     commands:
       - $CODEBUILD_SRC_DIR_toolbox/install.py
-      - xlz install drmlib_cred_json=~/.accelize accelpy_common_tf
-      - python3 -m pip install -Uq setuptools pip wheel pytest
+      - xlz install drmlib_cred_json=~/.accelize accelpy_common_tf accelpy_codecov_token
+      - python3 -m pip install -Uq setuptools pip wheel pytest-cov codecov
       - python3 -m pip install -qe .
 
   build:
     commands:
-      - ACCELPY_NO_COLOR=True pytest -m require_csp
+      - ACCELPY_NO_COLOR=True pytest -m require_csp --cov=accelpy --cov-report=term-missing
+      - codecov

docs/application_kubernetes_node.rst

@@ -52,6 +52,10 @@ This application support following variables:
 * `master_node`: If set to `true` (Default value), install the application as a
   single master/node. If set to `false`, install the application as a single
   node that must be integrated with an existing Kubernetes infrastructure.
+* `kubernetes_join_command`: Only if `master_node` is `false`. A command
+  to run on the node to joint the Kubernetes master. This command may be
+  generated by running `kubeadm token create --print-join-command` on the
+  master.
 
 Container configuration
 -----------------------

tests/app_kubernetes_pod-aws_f1.yml → tests/app_kubernetes_node-aws_f1.yml

@@ -35,3 +35,19 @@ spec:
         - name: sys
           hostPath:
             path: /sys
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kubernetes-node-test
+  labels:
+    name: kubernetes-node-test
+spec:
+  type: NodePort
+  selector:
+    app: kubernetes-node-test
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+      nodePort: 30080