Create artefact store object (#301)

docs/source/advanced_config.rst

@@ -315,7 +315,7 @@ which most Fab steps accept. (See :ref:`Overriding default collections`)
 
     @step
     def custom_step(state):
-            state._artefact_store['custom_artefacts'] = do_something(state._artefact_store['step 1 artefacts'])
+        state.artefact_store['custom_artefacts'] = do_something(state.artefact_store['step 1 artefacts'])
 
 
     with BuildConfig(project_label='<project label>') as state:
@@ -332,7 +332,7 @@ Steps have access to multiprocessing methods through the
 
     @step
     def custom_step(state):
-        input_files = artefact_store['custom_artefacts']
+        input_files = state.artefact_store['custom_artefacts']
         results = run_mp(state, items=input_files, func=do_something)
 
 

source/fab/artefacts.py

@@ -15,7 +15,7 @@
 from pathlib import Path
 from typing import Iterable, Union, Dict, List
 
-from fab.constants import BUILD_TREES
+from fab.constants import BUILD_TREES, CURRENT_PREBUILDS
 from fab.dep_tree import filter_source_tree, AnalysedDependent
 from fab.util import suffix_filter
 
@@ -32,7 +32,8 @@ def __call__(self, artefact_store):
             The artefact store from which to retrieve.
 
         """
-        pass
+        raise NotImplementedError(f"__call__ must be implemented for "
+                                  f"'{type(self).__name__}'.")
 
 
 class CollectionGetter(ArtefactsGetter):
@@ -53,7 +54,6 @@ def __init__(self, collection_name):
         self.collection_name = collection_name
 
     def __call__(self, artefact_store):
-        super().__call__(artefact_store)
         return artefact_store.get(self.collection_name, [])
 
 
@@ -84,7 +84,6 @@ def __init__(self, collections: Iterable[Union[str, ArtefactsGetter]]):
 
     # todo: ensure the labelled values are iterables
     def __call__(self, artefact_store: Dict):
-        super().__call__(artefact_store)
         # todo: this should be a set, in case a file appears in multiple collections
         result = []
         for collection in self.collections:
@@ -118,7 +117,6 @@ def __init__(self, collection_name: str, suffix: Union[str, List[str]]):
         self.suffixes = [suffix] if isinstance(suffix, str) else suffix
 
     def __call__(self, artefact_store):
-        super().__call__(artefact_store)
         # todo: returning an empty list is probably "dishonest" if the collection doesn't exist - return None instead?
         fpaths: Iterable[Path] = artefact_store.get(self.collection_name, [])
         return suffix_filter(fpaths, self.suffixes)
@@ -149,7 +147,6 @@ def __init__(self, suffix: Union[str, List[str]], collection_name: str = BUILD_T
         self.suffixes = [suffix] if isinstance(suffix, str) else suffix
 
     def __call__(self, artefact_store):
-        super().__call__(artefact_store)
 
         build_trees = artefact_store[self.collection_name]
 
@@ -158,3 +155,18 @@ def __call__(self, artefact_store):
             build_lists[root] = filter_source_tree(source_tree=tree, suffixes=self.suffixes)
 
         return build_lists
+
+
+class ArtefactStore(dict):
+    '''This object stores artefacts (which can be of any type). Each artefact
+    is indexed by a string.
+    '''
+    def __init__(self):
+        super().__init__()
+        self.reset()
+
+    def reset(self):
+        '''Clears the artefact store (but does not delete any files).
+        '''
+        self.clear()
+        self[CURRENT_PREBUILDS] = set()

source/fab/build_config.py

@@ -18,16 +18,18 @@
 from multiprocessing import cpu_count
 from pathlib import Path
 from string import Template
-from typing import List, Optional, Dict, Any, Iterable
+from typing import List, Optional, Iterable
 
+from fab.artefacts import ArtefactStore
 from fab.constants import BUILD_OUTPUT, SOURCE_ROOT, PREBUILD, CURRENT_PREBUILDS
 from fab.metrics import send_metric, init_metrics, stop_metrics, metrics_summary
+from fab.steps.cleanup_prebuilds import CLEANUP_COUNT, cleanup_prebuilds
 from fab.util import TimerLogger, by_type, get_fab_workspace
 
 logger = logging.getLogger(__name__)
 
 
-class BuildConfig(object):
+class BuildConfig():
     """
     Contains and runs a list of build steps.
 
@@ -105,9 +107,10 @@ def __init__(self, project_label: str, multiprocessing: bool = True, n_procs: Op
 
         # todo: should probably pull the artefact store out of the config
         # runtime
-        # todo: either make this public, add get/setters, or extract into a class.
-        self._artefact_store: Dict[str, Any] = {}
-        self.init_artefact_store()  # note: the artefact store is reset with every call to run()
+        self._artefact_store = ArtefactStore()
+
+        self._build_timer = None
+        self._start_time = None
 
     def __enter__(self):
 
@@ -130,8 +133,7 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
 
         if not exc_type:  # None if there's no error.
-            from fab.steps.cleanup_prebuilds import CLEANUP_COUNT, cleanup_prebuilds
-            if CLEANUP_COUNT not in self._artefact_store:
+            if CLEANUP_COUNT not in self.artefact_store:
                 logger.info("no housekeeping step was run, using a default hard cleanup")
                 cleanup_prebuilds(config=self, all_unused=True)
 
@@ -142,19 +144,23 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self._finalise_logging()
 
     @property
-    def build_output(self):
-        return self.project_workspace / BUILD_OUTPUT
+    def artefact_store(self) -> ArtefactStore:
+        ''':returns: the Artefact instance for this configuration.
+        '''
+        return self._artefact_store
 
-    def init_artefact_store(self):
-        # there's no point writing to this from a child process of Step.run_mp() because you'll be modifying a copy.
-        self._artefact_store = {CURRENT_PREBUILDS: set()}
+    @property
+    def build_output(self) -> Path:
+        ''':returns: the build output path.
+        '''
+        return self.project_workspace / BUILD_OUTPUT
 
     def add_current_prebuilds(self, artefacts: Iterable[Path]):
         """
         Mark the given file paths as being current prebuilds, not to be cleaned during housekeeping.
 
         """
-        self._artefact_store[CURRENT_PREBUILDS].update(artefacts)
+        self.artefact_store[CURRENT_PREBUILDS].update(artefacts)
 
     def _run_prep(self):
         self._init_logging()
@@ -168,7 +174,7 @@ def _run_prep(self):
         init_metrics(metrics_folder=self.metrics_folder)
 
         # note: initialising here gives a new set of artefacts each run
-        self.init_artefact_store()
+        self.artefact_store.reset()
 
     def _prep_folders(self):
         self.source_root.mkdir(parents=True, exist_ok=True)
@@ -210,7 +216,7 @@ def _finalise_metrics(self, start_time, steps_timer):
 
 
 # todo: better name? perhaps PathFlags?
-class AddFlags(object):
+class AddFlags():
     """
     Add command-line flags when our path filter matches.
     Generally used inside a :class:`~fab.build_config.FlagsConfig`.
@@ -265,14 +271,13 @@ def run(self, fpath: Path, input_flags: List[str], config):
             input_flags += add_flags
 
 
-class FlagsConfig(object):
+class FlagsConfig():
     """
     Return command-line flags for a given path.
 
     Simply allows appending flags but may evolve to also replace and remove flags.
 
     """
-
     def __init__(self, common_flags: Optional[List[str]] = None, path_flags: Optional[List[AddFlags]] = None):
         """
         :param common_flags:

source/fab/parse/fortran_common.py

@@ -134,8 +134,6 @@ def run(self, fpath: Path) \
 
         # find things in the node tree
         analysed_file = self.walk_nodes(fpath=fpath, file_hash=file_hash, node_tree=node_tree)
-
-        analysis_fpath = self._get_analysis_fpath(fpath, file_hash)
         analysed_file.save(analysis_fpath)
 
         return analysed_file, analysis_fpath

source/fab/steps/analyse.py

@@ -167,7 +167,7 @@ def analyse(
     c_analyser._config = config
 
     # parse
-    files: List[Path] = source_getter(config._artefact_store)
+    files: List[Path] = source_getter(config.artefact_store)
     analysed_files = _parse_files(config, files=files, fortran_analyser=fortran_analyser, c_analyser=c_analyser)
     _add_manual_results(special_measure_analysis_results, analysed_files)
 
@@ -206,7 +206,7 @@ def analyse(
         _add_unreferenced_deps(unreferenced_deps, symbol_table, project_source_tree, build_tree)
         validate_dependencies(build_tree)
 
-    config._artefact_store[BUILD_TREES] = build_trees
+    config.artefact_store[BUILD_TREES] = build_trees
 
 
 def _analyse_dependencies(analysed_files: Iterable[AnalysedDependent]):

source/fab/steps/archive_objects.py

@@ -95,14 +95,14 @@ def archive_objects(config: BuildConfig, source: Optional[ArtefactsGetter] = Non
     output_fpath = str(output_fpath) if output_fpath else None
     output_collection = output_collection
 
-    target_objects = source_getter(config._artefact_store)
+    target_objects = source_getter(config.artefact_store)
     assert target_objects.keys()
     if output_fpath and list(target_objects.keys()) != [None]:
         raise ValueError("You must not specify an output path (library) when there are root symbols (exes)")
     if not output_fpath and list(target_objects.keys()) == [None]:
         raise ValueError("You must specify an output path when building a library.")
 
-    output_archives = config._artefact_store.setdefault(output_collection, {})
+    output_archives = config.artefact_store.setdefault(output_collection, {})
     for root, objects in target_objects.items():
 
         if root:

source/fab/steps/c_pragma_injector.py

@@ -43,9 +43,9 @@ def c_pragma_injector(config, source: Optional[ArtefactsGetter] = None, output_n
     source_getter = source or DEFAULT_SOURCE_GETTER
     output_name = output_name or PRAGMAD_C
 
-    files = source_getter(config._artefact_store)
+    files = source_getter(config.artefact_store)
     results = run_mp(config, items=files, func=_process_artefact)
-    config._artefact_store[output_name] = list(results)
+    config.artefact_store[output_name] = list(results)
 
 
 def _process_artefact(fpath: Path):

source/fab/steps/cleanup_prebuilds.py

@@ -63,23 +63,23 @@ def cleanup_prebuilds(
 
     elif all_unused:
         num_removed = remove_all_unused(
-            found_files=prebuild_files, current_files=config._artefact_store[CURRENT_PREBUILDS])
+            found_files=prebuild_files, current_files=config.artefact_store[CURRENT_PREBUILDS])
 
     else:
         # get the file access time for every artefact
         prebuilds_ts = \
             dict(zip(prebuild_files, run_mp(config, prebuild_files, get_access_time)))  # type: ignore
 
         # work out what to delete
-        to_delete = by_age(older_than, prebuilds_ts, current_files=config._artefact_store[CURRENT_PREBUILDS])
-        to_delete |= by_version_age(n_versions, prebuilds_ts, current_files=config._artefact_store[CURRENT_PREBUILDS])
+        to_delete = by_age(older_than, prebuilds_ts, current_files=config.artefact_store[CURRENT_PREBUILDS])
+        to_delete |= by_version_age(n_versions, prebuilds_ts, current_files=config.artefact_store[CURRENT_PREBUILDS])
 
         # delete them all
         run_mp(config, to_delete, os.remove)
         num_removed = len(to_delete)
 
     logger.info(f'removed {num_removed} prebuild files')
-    config._artefact_store[CLEANUP_COUNT] = num_removed
+    config.artefact_store[CLEANUP_COUNT] = num_removed
 
 
 def by_age(older_than: Optional[timedelta],

source/fab/steps/compile_c.py

@@ -83,7 +83,7 @@ def compile_c(config, common_flags: Optional[List[str]] = None,
     source_getter = source or DEFAULT_SOURCE_GETTER
 
     # gather all the source to compile, for all build trees, into one big lump
-    build_lists: Dict = source_getter(config._artefact_store)
+    build_lists: Dict = source_getter(config.artefact_store)
     to_compile: list = sum(build_lists.values(), [])
     logger.info(f"compiling {len(to_compile)} c files")
 
@@ -101,7 +101,7 @@ def compile_c(config, common_flags: Optional[List[str]] = None,
     config.add_current_prebuilds(prebuild_files)
 
     # record the compilation results for the next step
-    store_artefacts(compiled_c, build_lists, config._artefact_store)
+    store_artefacts(compiled_c, build_lists, config.artefact_store)
 
 
 # todo: very similar code in fortran compiler

source/fab/steps/compile_fortran.py

@@ -84,7 +84,7 @@ def compile_fortran(config: BuildConfig, common_flags: Optional[List[str]] = Non
     mod_hashes: Dict[str, int] = {}
 
     # get all the source to compile, for all build trees, into one big lump
-    build_lists: Dict[str, List] = source_getter(config._artefact_store)
+    build_lists: Dict[str, List] = source_getter(config.artefact_store)
 
     # build the arguments passed to the multiprocessing function
     mp_common_args = MpCommonArgs(
@@ -119,7 +119,7 @@ def compile_fortran(config: BuildConfig, common_flags: Optional[List[str]] = Non
         logger.info(f"stage 2 compiled {len(compiled_this_pass)} files")
 
     # record the compilation results for the next step
-    store_artefacts(compiled, build_lists, config._artefact_store)
+    store_artefacts(compiled, build_lists, config.artefact_store)
 
 
 def handle_compiler_args(common_flags=None, path_flags=None):

source/fab/steps/find_source_files.py

@@ -145,4 +145,4 @@ def find_source_files(config, source_root=None, output_collection="all_source",
     if not filtered_fpaths:
         raise RuntimeError("no source files found after filtering")
 
-    config._artefact_store[output_collection] = filtered_fpaths
+    config.artefact_store[output_collection] = filtered_fpaths

source/fab/steps/link.py

@@ -78,11 +78,11 @@ def link_exe(config, linker: Optional[str] = None, flags=None, source: Optional[
     flags = flags or []
     source_getter = source or DefaultLinkerSource()
 
-    target_objects = source_getter(config._artefact_store)
+    target_objects = source_getter(config.artefact_store)
     for root, objects in target_objects.items():
         exe_path = config.project_workspace / f'{root}'
         call_linker(linker=linker, flags=flags, filename=str(exe_path), objects=objects)
-        config._artefact_store.setdefault(EXECUTABLES, []).append(exe_path)
+        config.artefact_store.setdefault(EXECUTABLES, []).append(exe_path)
 
 
 # todo: the bit about Dict[None, object_files] seems too obscure - try to rethink this.
@@ -123,7 +123,7 @@ def link_shared_object(config, output_fpath: str, linker: Optional[str] = None,
             flags.append(f)
 
     # We expect a single build target containing the whole codebase, with no name (as it's not a root symbol).
-    target_objects = source_getter(config._artefact_store)
+    target_objects = source_getter(config.artefact_store)
     assert list(target_objects.keys()) == [None]
 
     objects = target_objects[None]

source/fab/steps/preprocess.py

@@ -88,7 +88,7 @@ def pre_processor(config: BuildConfig, preprocessor: str,
     check_for_errors(results, caller_label=name)
 
     log_or_dot_finish(logger)
-    config._artefact_store[output_collection] = list(by_type(results, Path))
+    config.artefact_store[output_collection] = list(by_type(results, Path))
 
 
 def process_artefact(arg: Tuple[Path, MpCommonArgs]):
@@ -192,7 +192,7 @@ def preprocess_fortran(config: BuildConfig, source: Optional[ArtefactsGetter] =
 
     """
     source_getter = source or SuffixFilter('all_source', ['.F90', '.f90'])
-    source_files = source_getter(config._artefact_store)
+    source_files = source_getter(config.artefact_store)
     F90s = suffix_filter(source_files, '.F90')
     f90s = suffix_filter(source_files, '.f90')
 
@@ -257,7 +257,7 @@ def preprocess_c(config: BuildConfig, source=None, **kwargs):
 
     """
     source_getter = source or DefaultCPreprocessorSource()
-    source_files = source_getter(config._artefact_store)
+    source_files = source_getter(config.artefact_store)
 
     pre_processor(
         config,

source/fab/steps/psyclone.py

@@ -50,7 +50,7 @@ def preprocess_x90(config, common_flags: Optional[List[str]] = None):
         if fpp_flag not in common_flags:
             common_flags.append(fpp_flag)
 
-    source_files = SuffixFilter('all_source', '.X90')(config._artefact_store)
+    source_files = SuffixFilter('all_source', '.X90')(config.artefact_store)
 
     pre_processor(
         config,
@@ -132,7 +132,7 @@ def psyclone(config, kernel_roots: Optional[List[Path]] = None,
     source_getter = source_getter or DEFAULT_SOURCE_GETTER
     overrides_folder = overrides_folder
 
-    x90s = source_getter(config._artefact_store)
+    x90s = source_getter(config.artefact_store)
 
     # get the data for child processes to calculate prebuild hashes
     prebuild_analyses = _analysis_for_prebuilds(config, x90s, transformation_script, kernel_roots)
@@ -153,7 +153,7 @@ def psyclone(config, kernel_roots: Optional[List[Path]] = None,
     prebuild_files: List[Path] = list(chain(*by_type(prebuilds, List)))
 
     # record the output files in the artefact store for further processing
-    config._artefact_store['psyclone_output'] = output_files
+    config.artefact_store['psyclone_output'] = output_files
     outputs_str = "\n".join(map(str, output_files))
     logger.debug(f'psyclone outputs:\n{outputs_str}\n')