diff --git a/accelpy/__init__.py b/accelpy/__init__.py index a02abce..d9756d4 100644 --- a/accelpy/__init__.py +++ b/accelpy/__init__.py @@ -16,7 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -__version__ = '1.0.0-beta.19' +__version__ = '1.0.0-beta.20' __copyright__ = "Copyright 2019 Accelize" __licence__ = "Apache 2.0" diff --git a/accelpy/__main__.py b/accelpy/__main__.py index 485fceb..c8a86b5 100755 --- a/accelpy/__main__.py +++ b/accelpy/__main__.py @@ -268,12 +268,11 @@ def _application_completer(prefix, parsed_args, **__): # If not 100% sure the application is a local file, get applications from # the web service, but avoid to call it every time for performance reason. - # - "." may be found in paths likes "./" or as file extension delimiter, it - # cannot be found in the product_id, but may be found in the version - # (In this case, the delimiter ":" is also present). + # - Only path should starts with "." or "/" # - Product ID is in format "vendor/library/name" should not contain more # than 2 "/" - if ("." not in prefix or ':' in prefix) and prefix.count('/') <= 2: + if (not prefix.startswith('.') and not prefix.startswith('/') and + prefix.count('/') <= 2): from itertools import chain from accelpy._application import Application from accelpy.exceptions import ( diff --git a/accelpy/_ansible/roles/container_service/molecule/default/app_server.py b/accelpy/_ansible/roles/container_service/molecule/default/app_server.py index 4e2c5c6..c1be842 100755 --- a/accelpy/_ansible/roles/container_service/molecule/default/app_server.py +++ b/accelpy/_ansible/roles/container_service/molecule/default/app_server.py @@ -12,8 +12,9 @@ def do_GET(self): """GET""" process = run(['/opt/xilinx/xrt/bin/awssak', 'list'], stderr=STDOUT, stdout=PIPE) - self.send_response(500 if ( - process.returncode or b"[0] " not in process.stdout) else 200) + self.send_response( + 500 if (process.returncode or b"error" in process.stdout.lower()) + else 200) self.end_headers() self.wfile.write(process.stdout) diff --git a/accelpy/_ansible/roles/kubernetes_node/defaults/main.yml b/accelpy/_ansible/roles/kubernetes_node/defaults/main.yml index 30f154c..9ff9c97 100644 --- a/accelpy/_ansible/roles/kubernetes_node/defaults/main.yml +++ b/accelpy/_ansible/roles/kubernetes_node/defaults/main.yml @@ -1,3 +1,4 @@ --- single_node: true master_node: true +kubernetes_join_command: pwd # Do nothing, just to avoid crash diff --git a/accelpy/_application.py b/accelpy/_application.py index 911ff92..8244d72 100644 --- a/accelpy/_application.py +++ b/accelpy/_application.py @@ -1,5 +1,6 @@ # coding=utf-8 """Application Definition""" +from json import dumps from os import fsdecode from re import fullmatch @@ -89,10 +90,12 @@ class Application: definition (path-like object or dict): Path to yaml definition file or dict of the content of the definition. + configuration_id (int): ID of configuration in Accelize web service. """ - def __init__(self, definition): + def __init__(self, definition, configuration_id=None): self._providers = set() + self._configuration_id = configuration_id # Load from dict if isinstance(definition, dict): @@ -125,22 +128,35 @@ def from_id(cls, application): Args: application (str): Application if format "product_id:version" or - "product_id". + "product_id". If version is not specified, last stable version + available will be used. Returns: Application: Application definition. """ - # Get product ID and version + definition = cls._get_definition(application) + configuration_id = definition['application'].pop('configuration_id') + return cls(definition, configuration_id=configuration_id) + + @staticmethod + def _get_definition(application): + """ + Load application from Accelize web service. + + Args: + application (str): Application if format "product_id:version" or + "product_id". + + Returns: + dict: Raw definition from web service. + """ + params = dict(limit=1) try: - product_id, version = application.split(':', 1) + params['product_id'], params['version'] = application.split(':', 1) except ValueError: - product_id = application - version = None - - # Get definition from server - response = request.query('/auth/getapplicationdefinition/', - dict(product_id=product_id, version=version)) - return cls(response) + params['product_id'] = application + return request.query('/auth/objects/productconfiguration/', + params=params)['results'][0] @staticmethod def list(prefix=''): @@ -148,14 +164,14 @@ def list(prefix=''): List available applications on Accelize web service. Args: - product_id (str): Product ID linked to the application. prefix (str): Product ID prefix to filter. Returns: list of str: products. """ - return request.query('/auth/listapplicationdefinitions/', - dict(prefix=prefix)) + return request.query( + '/auth/objects/productconfigurationlistproduct/', params=dict( + product_id__startswith=prefix) if prefix else None)['results'] @staticmethod def list_versions(product_id, prefix=''): @@ -169,15 +185,49 @@ def list_versions(product_id, prefix=''): Returns: list of str: versions. """ - return request.query('/auth/listapplicationdefinitionversions/', - dict(product_id=product_id, prefix=prefix)) + params = dict(product_id=product_id) + if prefix: + params['version__startswith'] = prefix + return request.query('/auth/objects/productconfigurationlistversion/', + params=params)['results'] def push(self): """ Push application definition on Accelize web service. """ - return request.query('/auth/pushapplicationdefinition/', - self._definition, 'post') + self._configuration_id = request.query( + '/auth/objects/productconfiguration/', + data=dumps(self._clean_definition), + method='post')['application']['configuration_id'] + + @classmethod + def delete(cls, application): + """ + Delete application definition on Accelize web service. + + Args: + application (str or int): Application if format "product_id:version" + or "product_id" or configuration ID in Accelize web service. + If specific version of ID not specified, will delete the last + stable version. + """ + if not isinstance(application, int): + # Get configuration ID + application = cls._get_definition( + application)['application']['configuration_id'] + + request.query(f'/auth/objects/productconfiguration/{application}/', + method='delete') + + @property + def configuration_id(self): + """ + Configuration ID in Accelize web service. + + Returns: + int: ID + """ + return self._configuration_id @property def providers(self): @@ -249,7 +299,39 @@ def save(self, path=None): Args: path (path-like object): Path where save Yaml definition file. """ - yaml_write(self._definition, path or self._path) + yaml_write(self._clean_definition, path or self._path) + + @property + def _clean_definition(self): + """ + Return definition cleaned up from empty values. + + Returns: + dict: definition + """ + from copy import deepcopy + + definition = deepcopy(self._definition) + for section_name in tuple(definition): + section = definition[section_name] + + if isinstance(section, list): + for element in tuple(section): + for key in tuple(element): + if not element[key] and element[key] is not False: + del element[key] + if not element: + section.remove(element) + + else: + for key in tuple(section): + if not section[key] and section[key] is not False: + del section[key] + + if not section: + del definition[section_name] + + return definition def _validate(self, definition): """ diff --git a/accelpy/_common.py b/accelpy/_common.py index e8319fb..7ed7dc8 100644 --- a/accelpy/_common.py +++ b/accelpy/_common.py @@ -25,9 +25,6 @@ HOME_DIR = _expanduser('~/.accelize') CACHE_DIR = _join(HOME_DIR, '.cache') -#: Accelize endpoint -ACCELIZE_ENDPOINT = 'https://master.metering.accelize.com' - # Ensure directory exists and have restricted access rights _makesdirs(CACHE_DIR, exist_ok=True) _chmod(HOME_DIR, 0o700) @@ -423,11 +420,14 @@ class _Request: Request to accelize server. """ _TIMEOUT = 10 + _RETRIES = 3 + _ENDPOINT = 'https://master.metering.accelize.com' def __init__(self): self._token_expire = None self._token = None self._session = None + self._endpoint = self._ENDPOINT def _get_session(self): """ @@ -439,18 +439,28 @@ def _get_session(self): if self._session is None: # Lazy import, may never be called from requests import Session + from requests.adapters import HTTPAdapter + from urllib3.util.retry import Retry + + # Create session with automatic retries on some error codes + adapter = HTTPAdapter(max_retries=Retry( + total=self._RETRIES, read=self._RETRIES, connect=self._RETRIES, + backoff_factor=0.3, status_forcelist=(408, 500, 502, 504))) + self._session = Session() + self._session.mount('http://', adapter) + self._session.mount('https://', adapter) return self._session - def query(self, path, data=None, method='get'): + def query(self, path, method='get', **kwargs): """ Performs a query. Args: path (str): URL path - data (dict): data. method (str): Request method. + kwargs: Requests query kwargs. Returns: dict or list: Response. @@ -459,12 +469,13 @@ def query(self, path, data=None, method='get'): while True: # Get response + token = self._get_token() response = getattr(self._get_session(), method)( - ACCELIZE_ENDPOINT + path, data=data, - headers={"Authorization": "Bearer " + self._get_token(), - "Content-Type": "application/json", - "Accept": "application/vnd.accelize.v1+json"}, - timeout=self._TIMEOUT) + self._endpoint + path, headers={ + "Authorization": "Bearer " + token, + "Content-Type": "application/json", + "Accept": "application/vnd.accelize.v1+json"}, + timeout=self._TIMEOUT, **kwargs) # Token may be invalid if response.status_code == 401 and not retried: @@ -476,7 +487,10 @@ def query(self, path, data=None, method='get'): elif response.status_code >= 300: raise _ConfigurationException(self._get_error_message(response)) - return response.json() + try: + return response.json() + except _JSONDecodeError: + return def _get_error_message(self, response): """ @@ -489,7 +503,7 @@ def _get_error_message(self, response): str: Error message. """ try: - return response.json()["error"] + return response.json()["detail"] except (KeyError, _JSONDecodeError): return response.text @@ -514,6 +528,9 @@ def _get_token(self): client_id = credentials['client_id'] client_secret = credentials['client_secret'] + # Endpoint override in credentials file + self._endpoint = credentials.get('endpoint', self._ENDPOINT) + # Try to get from cache try: self._token, self._token_expire = get_cli_cache(client_id) @@ -521,7 +538,7 @@ def _get_token(self): # Try to get from web service except TypeError: response = self._get_session().post( - f'{ACCELIZE_ENDPOINT}/o/token/', + f'{self._endpoint}/o/token/', data={"grant_type": "client_credentials"}, auth=(client_id, client_secret), timeout=self._TIMEOUT) diff --git a/buildspec.yml b/buildspec.yml index c9a9fa6..20e8a66 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -12,10 +12,11 @@ phases: python: 3.7 commands: - $CODEBUILD_SRC_DIR_toolbox/install.py - - xlz install drmlib_cred_json=~/.accelize accelpy_common_tf - - python3 -m pip install -Uq setuptools pip wheel pytest + - xlz install drmlib_cred_json=~/.accelize accelpy_common_tf accelpy_codecov_token + - python3 -m pip install -Uq setuptools pip wheel pytest-cov codecov - python3 -m pip install -qe . build: commands: - - ACCELPY_NO_COLOR=True pytest -m require_csp + - ACCELPY_NO_COLOR=True pytest -m require_csp --cov=accelpy --cov-report=term-missing + - codecov diff --git a/docs/application_kubernetes_node.rst b/docs/application_kubernetes_node.rst index 95751b6..60bf82c 100644 --- a/docs/application_kubernetes_node.rst +++ b/docs/application_kubernetes_node.rst @@ -52,6 +52,10 @@ This application support following variables: * `master_node`: If set to `true` (Default value), install the application as a single master/node. If set to `false`, install the application as a single node that must be integrated with an existing Kubernetes infrastructure. +* `kubernetes_join_command`: Only if `master_node` is `false`. A command + to run on the node to joint the Kubernetes master. This command may be + generated by running `kubeadm token create --print-join-command` on the + master. Container configuration ----------------------- diff --git a/tests/app_kubernetes_pod-aws_f1.yml b/tests/app_kubernetes_node-aws_f1.yml similarity index 78% rename from tests/app_kubernetes_pod-aws_f1.yml rename to tests/app_kubernetes_node-aws_f1.yml index 8799b8a..4a6f5b6 100644 --- a/tests/app_kubernetes_pod-aws_f1.yml +++ b/tests/app_kubernetes_node-aws_f1.yml @@ -35,3 +35,19 @@ spec: - name: sys hostPath: path: /sys +--- +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-node-test + labels: + name: kubernetes-node-test +spec: + type: NodePort + selector: + app: kubernetes-node-test + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + nodePort: 30080 diff --git a/tests/app_kubernetes_service-aws_f1.yml b/tests/app_kubernetes_service-aws_f1.yml deleted file mode 100644 index ef4cf52..0000000 --- a/tests/app_kubernetes_service-aws_f1.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: kubernetes-node-test - labels: - name: kubernetes-node-test -spec: - type: NodePort - selector: - app: kubernetes-node-test - ports: - - protocol: TCP - port: 8080 - targetPort: 8080 - nodePort: 30080 diff --git a/tests/test_app_kubernetes_node.yml b/tests/test_app_kubernetes_node.yml index 55dd77b..d293328 100644 --- a/tests/test_app_kubernetes_node.yml +++ b/tests/test_app_kubernetes_node.yml @@ -16,10 +16,8 @@ fpga: count: 1 package: - - type: kubernetes_yaml - name: https://raw.githubusercontent.com/Accelize/accelpy/master/tests/app_kubernetes_pod-aws_f1.yml - - type: kubernetes_yaml - name: https://raw.githubusercontent.com/Accelize/accelpy/master/tests/app_kubernetes_service-aws_f1.yml + type: kubernetes_yaml + name: https://raw.githubusercontent.com/Accelize/accelpy/master/tests/app_kubernetes_node-aws_f1.yml accelize_drm: conf: diff --git a/tests/test_core_application.py b/tests/test_core_application.py index 22dac70..98d0993 100644 --- a/tests/test_core_application.py +++ b/tests/test_core_application.py @@ -414,4 +414,63 @@ def test_lint(tmpdir): image: image """) with pytest.raises(ConfigurationException): - Application(yml_file) \ No newline at end of file + Application(yml_file) + + +@pytest.mark.require_csp +def test_web_service_integration(): + """ + Test web service integration. + """ + from accelpy._common import request + from accelpy._application import Application + from random import randint + + # Use dev environment + request_endpoint = request._endpoint + request._endpoint = 'https://master.devmetering.accelize.com' + + product_id = 'accelize.com/accelpy/ci' + version = f'{randint(0, 255)}.{randint(0, 255)}.{randint(0, 255)}' + application = f'{product_id}:{version}' + + definition = dict( + application=dict( + product_id=product_id, type='container_service', version=version), + fpga=dict(image='nothing'), + package=dict(name='nothing', type='container_image'), + accelize_drm=dict(use_service=False)) + + try: + scr_app = Application(definition) + + # Test: push + scr_app.push() + + # Test: Get + srv_app = Application.from_id(application) + assert scr_app._definition == srv_app._definition + + # Test: List + assert product_id in Application.list() + + # Test: List with prefix + assert product_id in Application.list('accelize.com/accelpy') + + # Test: List version + assert version in Application.list_versions(product_id) + + # Test: List version with prefix + assert version in Application.list_versions( + product_id, version.split('.', 1)[0]) + + # Test: Delete + Application.delete(application) + assert version not in Application.list_versions(product_id) + + finally: + try: + Application.delete(application) + except Exception: + pass + request._endpoint = request_endpoint diff --git a/tests/test_provisioning.py b/tests/test_provisioning.py index 8451056..99efb69 100644 --- a/tests/test_provisioning.py +++ b/tests/test_provisioning.py @@ -41,7 +41,7 @@ def test_application(application_yml, tmpdir): tmpdir (py.path.local) tmpdir pytest fixture. """ from os import environ - from time import sleep + from time import sleep, time from subprocess import run, STDOUT, PIPE import accelpy._host as accelpy_host from accelpy._host import Host @@ -90,12 +90,16 @@ def test_application(application_yml, tmpdir): command = command.replace( shell_var, getattr(host, attr)) - # Run test command - sleep(5) - + # Run test command (With retries during 1 minute) print(f'\nRunning test command:\n{command.strip()}\n') - result = run(command, universal_newlines=True, - stderr=STDOUT, stdout=PIPE, shell=True) + timeout = time() + 60 + while time() < timeout: + result = run(command, universal_newlines=True, + stderr=STDOUT, stdout=PIPE, shell=True) + if not result.returncode: + break + sleep(1) + print(f'\nTest command returned: ' f'{result.returncode}\n{result.stdout}')