diff --git a/.bamboo/run_tests.sh b/.bamboo/run_tests.sh new file mode 100755 index 0000000..343a1ca --- /dev/null +++ b/.bamboo/run_tests.sh @@ -0,0 +1,12 @@ +export PATH=${bamboo_build_working_directory}/miniconda/bin:$PATH +export HOME=${bamboo_build_working_directory}/.home +export TMPDIR=${bamboo_build_working_directory} +export CONDA_PATH_BACKUP=${CONDA_PATH_BACKUP:-$PATH} +export CONDA_PREFIX=${CONDA_PREFIX:-} +export CONDA_PS1_BACKUP=${CONDA_PS1_BACKUP:-} + +source activate ${bamboo_build_working_directory}/.conda/conda_test_env +cd ${bamboo_build_working_directory} +coverage run --source ./ -m pytest --junitxml=test-reports/tests.xml --html=test-reports/report.html --self-contained-html +coverage html --omit="tests/*,setup.py" --directory=htmlcov +source deactivate diff --git a/.bamboo/setup_env.sh b/.bamboo/setup_env.sh new file mode 100755 index 0000000..897e331 --- /dev/null +++ b/.bamboo/setup_env.sh @@ -0,0 +1,16 @@ +set -eu + +wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh +bash miniconda.sh -b -p ${bamboo_build_working_directory}/miniconda +export PATH=${bamboo_build_working_directory}/miniconda/bin:$PATH +export HOME=${bamboo_build_working_directory}/.home +export TMPDIR=${bamboo_build_working_directory} +export CONDA_PATH_BACKUP=${CONDA_PATH_BACKUP:-$PATH} +export CONDA_PREFIX=${CONDA_PREFIX:-} +export CONDA_PS1_BACKUP=${CONDA_PS1_BACKUP:-} +conda create -y -v --prefix ${bamboo_build_working_directory}/.conda/conda_test_env python=3.6 +source activate ${bamboo_build_working_directory}/.conda/conda_test_env +cd ${bamboo_build_working_directory} +pip install -r requirements.txt +pip install -r requirements_dev.txt +source deactivate diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1c6d8b4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pyyaml +requests diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..af820b8 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,7 @@ +pytest +bumpversion +flake8 +pytest-cov +pytest-html +coverage +pandas \ No newline at end of file diff --git a/tests/utils.py b/tests/conftest.py similarity index 82% rename from tests/utils.py rename to tests/conftest.py index 11a7f8c..61cdf8d 100644 --- a/tests/utils.py +++ b/tests/conftest.py @@ -1,7 +1,9 @@ +import pytest import os import yaml -def load_regimen(): +@pytest.fixture +def regimen_dict(): regimen_path = os.path.join( os.path.dirname(__file__), '..', diff --git a/tests/test_downloadable.py b/tests/test_downloadable.py index a09ae7a..2dec56b 100644 --- a/tests/test_downloadable.py +++ b/tests/test_downloadable.py @@ -3,13 +3,47 @@ import hashlib import warnings import requests -from .utils import load_regimen +import sys +import itertools +import pandas as pd -regimen_data = load_regimen() +''' +Copy-pasted get_page and get_df; This is code duplication, but I dont want this to be an installed package, and both tests and scripts need this code +''' -def test_script_files_download(): +def get_page(table_name, api_base='http://prodmtrain1:5000', get_obj=None, **kwargs): - for stage_name, stage_dict in regimen_data['stages'].items(): + if get_obj is None: + get_obj = requests + + data = {'total_pages':'--'} + for ii in itertools.count(1): + print('Downloading page: %s/%s ' % (ii , data['total_pages']), end='') + sys.stdout.flush() + + tmp = get_obj.get(os.path.join(api_base, "api/v1/%s?page=%i" % (table_name, ii)), **kwargs) + try: + data = tmp.json() + except TypeError: + data = tmp.json + print('... ', end='') + if 'message' not in data: + df = pd.DataFrame(data["objects"]) + print('done') + sys.stdout.flush() + yield df + + if 'total_pages' not in data or data['total_pages'] == ii: + print('done') + return + +def get_df(table_name, api_base='http://prodmtrain1:5000', get_obj=None, **kwargs): + return pd.concat([df for df in get_page(table_name, api_base=api_base, get_obj=get_obj, **kwargs)], axis=0) + + +def test_script_files_download(regimen_dict): + + for stage_name, stage_dict in regimen_dict['stages'].items(): if stage_dict['script'] == 'NotImplemented': warnings.warn('Stage "%s" Not Implemented' % stage_name) else: @@ -25,12 +59,9 @@ def test_script_files_download(): downloaded_md5 = hashlib.md5(result.content).hexdigest() assert downloaded_md5 == stage_dict['script_md5'], "{}:{}".format(stage_name,downloaded_md5) -def test_add_already_tracked_regimen(): +def test_add_already_tracked_regimen(regimen_dict): - api_endpoint = 'http://prodmtrain1:5000/api/v1/regimens' - result = requests.get(api_endpoint) - assert result.status_code == 200 - data = result.json() - assert data['total_pages'] >= 1 - if regimen_data['name'] in [x['name'] for x in data['objects']]: - warnings.warn('Regimen name %s already tracked on server %s' % (regimen_data['name'], api_endpoint)) + api_base = 'http://prodmtrain1:5000' + regimen_list = get_df('regimens', api_base=api_base)['name'].values + if regimen_dict['name'] in regimen_list: + raise Exception('Regimen name %s already tracked on server %s' % (regimen_dict['name'], api_base)) diff --git a/tests/test_stage_name_in_params.py b/tests/test_stage_name_in_params.py index 43e58aa..4ac31db 100644 --- a/tests/test_stage_name_in_params.py +++ b/tests/test_stage_name_in_params.py @@ -1,8 +1,16 @@ -from .utils import load_regimen -regimen_data = load_regimen() - -def test_stage_name_in_params(): - for stage_name, stage_dict in regimen_data['stages'].items(): +def test_stage_name_in_params(regimen_dict): + for stage_name, stage_dict in regimen_dict['stages'].items(): stage_param = stage_dict['parameters']['stage'] assert stage_param==stage_name, "stage: {}, param: {}".format(stage_name,stage_param) + +def test_transitions_stages_match(regimen_dict): + + # Double-check regimen stages and transitions match: + transition_source_target_set = set() + for t in regimen_dict['transitions']: + transition_source_target_set.add(t['source']) + transition_source_target_set.add(t['dest']) + stage_set = set(regimen_dict['stages'].keys()) + if not all([curr_stage in stage_set for curr_stage in transition_source_target_set]): + raise Exception('Stages and source/dest of transitions are not consistent') \ No newline at end of file