diff --git a/.pylintrc b/.pylintrc index 61d1bddc0..117b83489 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,5 @@ [FORMAT] -max-line-length=150 +max-line-length=120 good-names=f, _log [TYPECHECK] ignored-modules = numpy, numpy.random diff --git a/requirements.txt b/requirements.txt index faeec073e..cc6325d0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ graphviz numpy pandas mock -pyaml +pyyaml scipy diff --git a/tests/framework/test_components.py b/tests/framework/test_components.py index 7f7c50850..73483bd1c 100644 --- a/tests/framework/test_components.py +++ b/tests/framework/test_components.py @@ -1,12 +1,15 @@ -import pytest import sys import os -import yaml import ast from unittest.mock import patch, mock_open -from vivarium.framework.components import _import_by_path, load_component_manager, ComponentManager, DummyDatasetManager, ComponentConfigError, _extract_component_list, _component_ast_to_path, _parse_component, ParsingError, _prep_components -from vivarium import config +import pytest +import yaml + +from vivarium.framework.components import (_import_by_path, load_component_manager, ComponentManager, + DummyDatasetManager, ComponentConfigError, _extract_component_list, + _component_ast_to_path, _parse_component, ParsingError, _prep_components) + # Fiddle the path so we can import from this module sys.path.append(os.path.dirname(__file__)) @@ -33,20 +36,24 @@ dataset_manager: test_components.MockDatasetManager """ + class MockComponentManager: def __init__(self, components, dataset_manager): self.components = components self.dataset_manager = dataset_manager + class MockDatasetManager: def __init__(self): self.constructors = {} + class MockComponentA: def __init__(self, *args): self.args = args self.builder_used_for_setup = None + class MockComponentB(MockComponentA): def setup(self, builder): self.builder_used_for_setup = builder @@ -57,27 +64,33 @@ def setup(self, builder): children.append(MockComponentB(arg)) return children + def mock_component_c(): pass + # This very strange import makes it so the classes in the current scope have the same # absolute paths as the ones my tests will cause the tools to import so I can compare them. from test_components import MockComponentA, MockComponentB, mock_component_c, MockDatasetManager, MockComponentManager + def test_import_class_by_path(): cls = _import_by_path('collections.abc.Set') from collections.abc import Set assert cls is Set + def test_import_function_by_path(): func = _import_by_path('vivarium.framework.components._import_by_path') assert func is _import_by_path + def test_bad_import_by_path(): with pytest.raises(ImportError): - cls = _import_by_path('junk.garbage.SillyClass') + _import_by_path('junk.garbage.SillyClass') with pytest.raises(AttributeError): - cls = _import_by_path('vivarium.framework.components.SillyClass') + _import_by_path('vivarium.framework.components.SillyClass') + def test_load_component_manager_defaults(): manager = load_component_manager(config_source=TEST_COMPONENTS+TEST_CONFIG_DEFAULTS) @@ -85,27 +98,34 @@ def test_load_component_manager_defaults(): assert isinstance(manager, ComponentManager) assert isinstance(manager.dataset_manager, DummyDatasetManager) + def test_load_component_manager_custom_managers(): - manager = load_component_manager(config_source=TEST_COMPONENTS+TEST_CONFIG_DEFAULTS+TEST_CONFIG_CUSTOM_COMPONENT_MANAGER) + manager = load_component_manager( + config_source=TEST_COMPONENTS + TEST_CONFIG_DEFAULTS + TEST_CONFIG_CUSTOM_COMPONENT_MANAGER) assert isinstance(manager, MockComponentManager) - manager = load_component_manager(config_source=TEST_COMPONENTS+TEST_CONFIG_DEFAULTS+TEST_CONFIG_CUSTOM_DATASET_MANAGER) + manager = load_component_manager( + config_source=TEST_COMPONENTS + TEST_CONFIG_DEFAULTS + TEST_CONFIG_CUSTOM_DATASET_MANAGER) assert isinstance(manager.dataset_manager, MockDatasetManager) -@patch('vivarium.framework.components.open', mock_open(read_data=TEST_COMPONENTS+TEST_CONFIG_DEFAULTS+TEST_CONFIG_CUSTOM_COMPONENT_MANAGER)) + +@patch('vivarium.framework.components.open', + mock_open(read_data=TEST_COMPONENTS+TEST_CONFIG_DEFAULTS+TEST_CONFIG_CUSTOM_COMPONENT_MANAGER)) def test_load_component_manager_path(): manager = load_component_manager(config_path='/etc/ministries/conf.d/silly_walk.yaml') assert isinstance(manager, MockComponentManager) + def test_load_component_manager_errors(): with pytest.raises(ComponentConfigError): - manager = load_component_manager() + load_component_manager() with pytest.raises(ComponentConfigError): - manager = load_component_manager(config_source='{}', config_path='/test.yaml') + load_component_manager(config_source='{}', config_path='/test.yaml') with pytest.raises(ComponentConfigError): - manager = load_component_manager(config_path='/test.json') + load_component_manager(config_path='/test.json') + def test_extract_component_list(): source = yaml.load(TEST_COMPONENTS)['components'] @@ -116,15 +136,23 @@ def test_extract_component_list(): 'ministry.silly_walk.PratFall(15)', 'pet_shop.Parrot()'} == set(components) + def test_component_ast_to_path(): - call, *args = ast.iter_child_nodes(list(ast.iter_child_nodes(list(ast.iter_child_nodes(ast.parse('cave_system.monsters.Rabbit()')))[0]))[0]) + call, *args = ast.iter_child_nodes( + list(ast.iter_child_nodes( + list(ast.iter_child_nodes( + ast.parse('cave_system.monsters.Rabbit()')))[0]))[0]) path = _component_ast_to_path(call) assert path == 'cave_system.monsters.Rabbit' - call, *args = ast.iter_child_nodes(list(ast.iter_child_nodes(list(ast.iter_child_nodes(ast.parse('Rabbit()')))[0]))[0]) + call, *args = ast.iter_child_nodes( + list(ast.iter_child_nodes( + list(ast.iter_child_nodes( + ast.parse('Rabbit()')))[0]))[0]) path = _component_ast_to_path(call) assert path == 'Rabbit' + def test_parse_component(): constructors = {'Dentition': lambda tooth_count: '{} teeth'.format(tooth_count)} @@ -133,9 +161,10 @@ def test_parse_component(): assert component == 'cave_system.monsters.Rabbit' assert set(args) == {'timid', 0.01} - desc = 'cave_system.monsters.Rabbit("ravinous", 10, Dentition("102"))' + desc = 'cave_system.monsters.Rabbit("ravenous", 10, Dentition("102"))' component, args = _parse_component(desc, constructors) - assert set(args) == {'ravinous', '102 teeth', 10} + assert set(args) == {'ravenous', '102 teeth', 10} + def test_parse_component_syntax_error(): # No non-literal arguments that aren't handled by constructors @@ -148,6 +177,7 @@ def test_parse_component_syntax_error(): desc = 'village.people.PlagueVictim(Causes(np.array(["black_death", "helminth"])))' _parse_component(desc, {'Causes': lambda cs: list(cs)}) + def test_prep_components(): component_descriptions = [ 'test_components.MockComponentA(Placeholder("A Hundred and One Ways to Start a Fight"))', @@ -156,7 +186,7 @@ def test_prep_components(): ] components = _prep_components(component_descriptions, {'Placeholder': lambda x: x}) - components = {c[0]:c[1] if len(c) == 2 else None for c in components} + components = {c[0]: c[1] if len(c) == 2 else None for c in components} assert len(components) == 3 assert MockComponentA in components @@ -164,12 +194,14 @@ def test_prep_components(): assert mock_component_c in components assert components[MockComponentA] == ['A Hundred and One Ways to Start a Fight'] assert components[MockComponentB] == ['Ethel the Aardvark goes Quantity Surveying'] - assert components[mock_component_c] == None + assert components[mock_component_c] is None + @patch('vivarium.framework.components._extract_component_list') @patch('vivarium.framework.components._prep_components') def test_ComponentManager__load_component_from_config(_prep_components_mock, _extract_component_list_mock): - _prep_components_mock.return_value = [(MockComponentA, ['Red Leicester']), (MockComponentB, []), (mock_component_c,)] + _prep_components_mock.return_value = [(MockComponentA, ['Red Leicester']), + (MockComponentB, []), (mock_component_c,)] manager = ComponentManager({}, MockDatasetManager()) @@ -178,11 +210,12 @@ def test_ComponentManager__load_component_from_config(_prep_components_mock, _ex assert len(manager.components) == 3 assert mock_component_c in manager.components - mocka = [c for c in manager.components if c.__class__ == MockComponentA][0] - mockb = [c for c in manager.components if c.__class__ == MockComponentB][0] + mock_a = [c for c in manager.components if c.__class__ == MockComponentA][0] + mock_b = [c for c in manager.components if c.__class__ == MockComponentB][0] + + assert list(mock_a.args) == ['Red Leicester'] + assert list(mock_b.args) == [] - assert list(mocka.args) == ['Red Leicester'] - assert list(mockb.args) == [] def test_ComponentManager__setup_components(): manager = ComponentManager({}, MockDatasetManager()) @@ -192,14 +225,14 @@ def test_ComponentManager__setup_components(): manager.setup_components(builder) - mocka, mockb, mockc, mockb_child1, mockb_child2, mockb_child3 = manager.components - - assert mocka.builder_used_for_setup is None # class has no setup method - assert mockb.builder_used_for_setup is builder - assert mockc is mock_component_c - assert mockb_child1.args == ('half',) - assert mockb_child1.builder_used_for_setup is builder - assert mockb_child2.args == ('a',) - assert mockb_child2.builder_used_for_setup is builder - assert mockb_child3.args == ('bee',) - assert mockb_child3.builder_used_for_setup is builder + mock_a, mock_b, mock_c, mock_b_child1, mock_b_child2, mock_b_child3 = manager.components + + assert mock_a.builder_used_for_setup is None # class has no setup method + assert mock_b.builder_used_for_setup is builder + assert mock_c is mock_component_c + assert mock_b_child1.args == ('half',) + assert mock_b_child1.builder_used_for_setup is builder + assert mock_b_child2.args == ('a',) + assert mock_b_child2.builder_used_for_setup is builder + assert mock_b_child3.args == ('bee',) + assert mock_b_child3.builder_used_for_setup is builder diff --git a/vivarium/framework/celery_tasks.py b/vivarium/framework/celery_tasks.py index 3c92facb7..81efb05eb 100644 --- a/vivarium/framework/celery_tasks.py +++ b/vivarium/framework/celery_tasks.py @@ -11,16 +11,17 @@ app = Celery() + @app.task(autoretry_for=(Exception,), max_retries=2) def worker(input_draw_number, model_draw_number, component_config, branch_config, logging_directory): np.random.seed([input_draw_number, model_draw_number]) - worker = current_process().index - logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', filename=os.path.join(logging_directory, str(worker)+'.log'), level=logging.DEBUG) + worker_ = current_process().index + logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + filename=os.path.join(logging_directory, str(worker_)+'.log'), level=logging.DEBUG) logging.info('Starting job: {}'.format((input_draw_number, model_draw_number, component_config, branch_config))) run_configuration = component_config['configuration'].get('run_configuration', {}) - results_directory = run_configuration['results_directory'] - run_configuration['run_id'] = str(worker)+'_'+str(time()) + run_configuration['run_id'] = str(worker_)+'_'+str(time()) if branch_config is not None: run_configuration['run_key'] = dict(branch_config) run_configuration['run_key']['input_draw'] = input_draw_number @@ -32,14 +33,16 @@ def worker(input_draw_number, model_draw_number, component_config, branch_config from vivarium.framework.components import load_component_manager from vivarium.framework.util import collapse_nested_dict - configure(input_draw_number=input_draw_number, model_draw_number=model_draw_number, simulation_config=branch_config) - component_maneger = load_component_manager(component_config) + configure(input_draw_number=input_draw_number, + model_draw_number=model_draw_number, simulation_config=branch_config) + component_manager = load_component_manager(component_config) results = run(component_manager) - idx=pd.MultiIndex.from_tuples([(input_draw_number, model_draw_number)], names=['input_draw_number','model_draw_number']) + idx = pd.MultiIndex.from_tuples([(input_draw_number, model_draw_number)], + names=['input_draw_number', 'model_draw_number']) results = pd.DataFrame(results, index=idx).to_json() return results - except Exception as e: + except Exception: logging.exception('Unhandled exception in worker') raise finally: diff --git a/vivarium/framework/components.py b/vivarium/framework/components.py index 57b7259ce..6f93ffe26 100644 --- a/vivarium/framework/components.py +++ b/vivarium/framework/components.py @@ -1,7 +1,6 @@ -"""Tools for interpreting component configuration files as well as the default ComponentManager class which uses those tools -to load and manage components. +"""Tools for interpreting component configuration files as well as the default +ComponentManager class which uses those tools to load and manage components. """ - import ast from collections import Iterable from importlib import import_module @@ -14,20 +13,17 @@ class ComponentConfigError(Exception): - """Error while interpreting configuration file or initializing components - """ + """Error while interpreting configuration file or initializing components""" pass class ParsingError(ComponentConfigError): - """Error while parsing component descriptions - """ + """Error while parsing component descriptions""" pass class DummyDatasetManager: - """Placeholder implementation of the DatasetManager - """ + """Placeholder implementation of the DatasetManager""" def __init__(self): self.constructors = {} @@ -46,8 +42,9 @@ def _import_by_path(path: str) -> Union[type, Callable]: def load_component_manager(config_source: str = None, config_path: str = None, dataset_manager_class: type = None): - """Create a component manager along with it's dataset manager. The class used will be either the default or - a custom class specified in the configuration. + """Create a component manager along with it's dataset manager. + + The class used will be either the default or a custom class specified in the configuration. Parameters ---------- @@ -56,7 +53,8 @@ def load_component_manager(config_source: str = None, config_path: str = None, d config_path: The path to a YAML configuration file to use. dataset_manager_class: - Class to use for the dataset manager. Will override dataset manager class specified in the configuration if supplied. + Class to use for the dataset manager. Will override dataset manager + class specified in the configuration if supplied. """ if sum([config_source is None, config_path is None]) != 1: @@ -102,7 +100,6 @@ def __init__(self, component_config, dataset_manager): self.components = [] self.dataset_manager = dataset_manager - def load_components_from_config(self): """Load and initialize (if necessary) any components listed in the config and register them with the ComponentManager. """ @@ -119,7 +116,6 @@ def load_components_from_config(self): self.components.extend(new_components) - def add_components(self, components: Sequence): """Register new components. @@ -131,7 +127,6 @@ def add_components(self, components: Sequence): self.components = components + self.components - def setup_components(self, builder): """Apply component level configuration defaults to the global config and run setup methods on the components registering and setting up any child components generated in the process. @@ -190,9 +185,9 @@ def _process_level(level, prefix): return _process_level(component_config, []) + def _component_ast_to_path(component: ast.AST) -> str: - """Convert the AST representing a component into a string - which can be imported. + """Convert the AST representing a component into a string which can be imported. Parameters ---------- @@ -210,6 +205,7 @@ def _component_ast_to_path(component: ast.AST) -> str: path.insert(0, current.id) return '.'.join(path) + def _parse_component(desc: str, constructors: Mapping[str, Callable]) -> Tuple[str, Sequence]: """Parse a component definition in a subset of python syntax and return an importable path to the specified component along with the arguments it should receive when invoked. @@ -227,7 +223,8 @@ def _parse_component(desc: str, constructors: Mapping[str, Callable]) -> Tuple[s Dictionary of callables for creating argument objects """ - component, *args = ast.iter_child_nodes(list(ast.iter_child_nodes(list(ast.iter_child_nodes(ast.parse(desc)))[0]))[0]) + component, *args = ast.iter_child_nodes(list(ast.iter_child_nodes( + list(ast.iter_child_nodes(ast.parse(desc)))[0]))[0]) component = _component_ast_to_path(component) new_args = [] for arg in args: @@ -239,7 +236,8 @@ def _parse_component(desc: str, constructors: Mapping[str, Callable]) -> Tuple[s if isinstance(arg, ast.Call): constructor, *constructor_args = ast.iter_child_nodes(arg) constructor = constructors.get(constructor.id) - # NOTE: This currently precludes arguments other than strings. May want to release that constraint later. + # NOTE: This currently precludes arguments other than strings. + # May want to release that constraint later. if constructor and len(constructor_args) == 1 and isinstance(constructor_args[0], ast.Str): new_args.append(constructor(constructor_args[0].s)) parsed = True @@ -249,8 +247,9 @@ def _parse_component(desc: str, constructors: Mapping[str, Callable]) -> Tuple[s raise ParsingError('Invalid syntax: {}'.format(desc)) return component, new_args + def _prep_components(component_list: Sequence, constructors: Mapping[str, Callable]) -> Sequence: - """Transform component description strings into tuples of component callables and any arguments the component may need. + """Transform component description strings into tuples of component callables and arguments the component may need. Parameters ---------- diff --git a/vivarium/framework/engine.py b/vivarium/framework/engine.py index fcf666dba..4ed39a777 100644 --- a/vivarium/framework/engine.py +++ b/vivarium/framework/engine.py @@ -63,7 +63,6 @@ def setup(self): self.component_manager.load_components_from_config() self.component_manager.setup_components(builder) - self.values.setup_components(self.component_manager.components) self.events.setup_components(self.component_manager.components) self.population.setup_components(self.component_manager.components) @@ -238,5 +237,6 @@ def main(): logging.exception("Uncaught exception {}".format(e)) raise + if __name__ == '__main__': main()