Skip to content

Commit

Permalink
Restyle repository to 80 columns.
Browse files Browse the repository at this point in the history
  • Loading branch information
@MatthewHambley
MatthewHambley committed Jul 10, 2024
1 parent 3452576 commit 1f2f5a5
Show file tree
Hide file tree
Showing 60 changed files with 2,533 additions and 1,193 deletions.
3 changes: 0 additions & 3 deletions .flake8
View file

This file was deleted.

51 changes: 29 additions & 22 deletions source/fab/artefacts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@
# which you should have received as part of this distribution
##############################################################################
"""
This module contains :term:`Artefacts Getter` classes which return :term:`Artefact Collections <Artefact Collection>`
from the :term:`Artefact Store`.
These classes are used by the `run` method of :class:`~fab.steps.Step` classes to retrieve the artefacts
which need to be processed. Most steps have sensible defaults and can be configured with user-defined getters.
This module contains :term:`Artefacts Getter` classes which return
:term:`Artefact Collections <Artefact Collection>` from the
:term:`Artefact Store`.
These classes are used by the `run` method of :class:`~fab.steps.Step` classes
to retrieve the artefacts which need to be processed. Most steps have sensible
defaults and can be configured with user-defined getters.
"""
from abc import ABC, abstractmethod
from pathlib import Path
Expand Down Expand Up @@ -53,12 +54,12 @@ def __call__(self, artefact_store):

class CollectionGetter(ArtefactsGetter):
"""
A simple artefact getter which returns one :term:`Artefact Collection` from the artefact_store.
A simple artefact getter which returns one :term:`Artefact Collection`
from the artefact_store.
Example::
`CollectionGetter('preprocessed_fortran')`
"""
def __init__(self, collection_name):
"""
Expand All @@ -74,32 +75,34 @@ def __call__(self, artefact_store):

class CollectionConcat(ArtefactsGetter):
"""
Returns a concatenated list from multiple :term:`Artefact Collections <Artefact Collection>`
(each expected to be an iterable).
Returns a concatenated list from multiple :term:`Artefact Collections
<Artefact Collection> (each expected to be an iterable).
An :class:`~fab.artefacts.ArtefactsGetter` can be provided instead of a collection_name.
An :class:`~fab.artefacts.ArtefactsGetter` can be provided instead of a
collection_name.
Example::
# The default source code getter for the Analyse step might look like this.
# The default source code getter for the Analyse step might look like
# this.
DEFAULT_SOURCE_GETTER = CollectionConcat([
'preprocessed_c',
'preprocessed_fortran',
SuffixFilter('all_source', '.f90'),
])
"""
def __init__(self, collections: Iterable[Union[str, ArtefactsGetter]]):
"""
:param collections:
An iterable containing collection names (strings) or other ArtefactsGetters.
An iterable containing collection names (strings) or other
ArtefactsGetters.
"""
self.collections = collections

# todo: ensure the labelled values are iterables
def __call__(self, artefact_store: ArtefactStore):
# todo: this should be a set, in case a file appears in multiple collections
# todo: this should be a set, in case a file appears in multiple
# collections
result = []
for collection in self.collections:
if isinstance(collection, str):
Expand All @@ -111,14 +114,13 @@ def __call__(self, artefact_store: ArtefactStore):

class SuffixFilter(ArtefactsGetter):
"""
Returns the file paths in a :term:`Artefact Collection` (expected to be an iterable),
filtered by suffix.
Returns the file paths in a :term:`Artefact Collection` (expected to be an
iterable), filtered by suffix.
Example::
# The default source getter for the FortranPreProcessor step.
DEFAULT_SOURCE = SuffixFilter('all_source', '.F90')
"""
def __init__(self, collection_name: str, suffix: Union[str, List[str]]):
"""
Expand All @@ -132,7 +134,8 @@ def __init__(self, collection_name: str, suffix: Union[str, List[str]]):
self.suffixes = [suffix] if isinstance(suffix, str) else suffix

def __call__(self, artefact_store: ArtefactStore):
# todo: returning an empty list is probably "dishonest" if the collection doesn't exist - return None instead?
# todo: returning an empty list is probably "dishonest" if the
# collection doesn't exist - return None instead?
fpaths: Iterable[Path] = artefact_store.get(self.collection_name, [])
return suffix_filter(fpaths, self.suffixes)

Expand All @@ -141,15 +144,18 @@ class FilterBuildTrees(ArtefactsGetter):
"""
Filter build trees by suffix.
Returns one list of files to compile per build tree, of the form Dict[name, List[AnalysedDependent]]
Returns one list of files to compile per build tree, of the form
Dict[name, List[AnalysedDependent]]
Example::
# The default source getter for the CompileFortran step.
DEFAULT_SOURCE_GETTER = FilterBuildTrees(suffix='.f90')
"""
def __init__(self, suffix: Union[str, List[str]], collection_name: str = BUILD_TREES):
def __init__(self,
suffix: Union[str, List[str]],
collection_name: str = BUILD_TREES):
"""
:param suffix:
A suffix string, or iterable of, including the preceding dot.
Expand All @@ -167,6 +173,7 @@ def __call__(self, artefact_store: ArtefactStore):

build_lists: Dict[str, List[AnalysedDependent]] = {}
for root, tree in build_trees.items():
build_lists[root] = filter_source_tree(source_tree=tree, suffixes=self.suffixes)
build_lists[root] = filter_source_tree(source_tree=tree,
suffixes=self.suffixes)

return build_lists
96 changes: 63 additions & 33 deletions source/fab/build_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,14 @@
from typing import List, Optional, Iterable

from fab.artefacts import ArtefactStore
from fab.constants import BUILD_OUTPUT, SOURCE_ROOT, PREBUILD, CURRENT_PREBUILDS
from fab.metrics import send_metric, init_metrics, stop_metrics, metrics_summary
from fab.constants import (BUILD_OUTPUT,
SOURCE_ROOT,
PREBUILD,
CURRENT_PREBUILDS)
from fab.metrics import (send_metric,
init_metrics,
stop_metrics,
metrics_summary)
from fab.tools.category import Category
from fab.tools.tool_box import ToolBox
from fab.steps.cleanup_prebuilds import CLEANUP_COUNT, cleanup_prebuilds
Expand All @@ -47,25 +53,28 @@ def __init__(self, project_label: str,
verbose=False):
"""
:param project_label:
Name of the build project. The project workspace folder is created from this name, with spaces replaced
by underscores.
Name of the build project. The project workspace folder is created
from this name, with spaces replaced by underscores.
:param tool_box: The ToolBox with all tools to use in the build.
:param multiprocessing:
An option to disable multiprocessing to aid debugging.
:param n_procs:
The number of cores to use for multiprocessing operations. Defaults to the number of available cores.
The number of cores to use for multiprocessing operations.
Defaults to the number of available cores.
:param reuse_artefacts:
A flag to avoid reprocessing certain files on subsequent runs.
WARNING: Currently unsophisticated, this flag should only be used by Fab developers.
The logic behind flag will soon be improved, in a work package called "incremental build".
WARNING: Currently unsophisticated, this flag should only be used
by Fab developers. The logic behind flag will soon be improved, in
a work package called "incremental build".
:param fab_workspace:
Overrides the FAB_WORKSPACE environment variable.
If not set, and FAB_WORKSPACE is not set, the fab workspace defaults to *~/fab-workspace*.
If not set, and FAB_WORKSPACE is not set, the fab workspace
defaults to *~/fab-workspace*.
:param two_stage:
Compile .mod files first in a separate pass. Theoretically faster in some projects..
Compile .mod files first in a separate pass. Theoretically faster
in some projects.
:param verbose:
DEBUG level logging.
"""
self._tool_box = tool_box
self.two_stage = two_stage
Expand All @@ -83,7 +92,8 @@ def __init__(self, project_label: str,
logger.info(f"fab workspace is {fab_workspace}")

self.project_workspace: Path = fab_workspace / self.project_label
self.metrics_folder: Path = self.project_workspace / 'metrics' / self.project_label
self.metrics_folder = self.project_workspace \
/ 'metrics' / self.project_label

# source config
self.source_root: Path = self.project_workspace / SOURCE_ROOT
Expand All @@ -93,7 +103,8 @@ def __init__(self, project_label: str,
self.multiprocessing = multiprocessing

# turn off multiprocessing when debugging
# todo: turn off multiprocessing when running tests, as a good test runner will run using mp
# todo: turn off multiprocessing when running tests, as a good test
# runner will run using mp
if 'pydevd' in str(sys.gettrace()):
logger.info('debugger detected, running without multiprocessing')
self.multiprocessing = False
Expand Down Expand Up @@ -129,7 +140,9 @@ def __enter__(self):
self._start_time = datetime.now().replace(microsecond=0)
self._run_prep()

with TimerLogger(f'running {self.project_label} build steps') as build_timer:
with TimerLogger(
f'running {self.project_label} build steps'
) as build_timer:
# this will return to the build script
self._build_timer = build_timer
return self
Expand All @@ -138,10 +151,14 @@ def __exit__(self, exc_type, exc_val, exc_tb):

if not exc_type: # None if there's no error.
if CLEANUP_COUNT not in self.artefact_store:
logger.info("no housekeeping step was run, using a default hard cleanup")
logger.info("no housekeeping step was run, "
"using a default hard cleanup")
cleanup_prebuilds(config=self, all_unused=True)

logger.info(f"Building '{self.project_label}' took {datetime.now() - self._start_time}")
logger.info(
f"Building '{self.project_label}' took "
+ str(datetime.now() - self._start_time)
)

# always
self._finalise_metrics(self._start_time, self._build_timer)
Expand All @@ -166,8 +183,8 @@ def build_output(self) -> Path:

def add_current_prebuilds(self, artefacts: Iterable[Path]):
"""
Mark the given file paths as being current prebuilds, not to be cleaned during housekeeping.
Mark the given file paths as being current prebuilds, not to be
cleaned during housekeeping.
"""
self.artefact_store[CURRENT_PREBUILDS].update(artefacts)

Expand All @@ -193,7 +210,9 @@ def _prep_folders(self):
def _init_logging(self):
# add a file logger for our run
self.project_workspace.mkdir(parents=True, exist_ok=True)
log_file_handler = RotatingFileHandler(self.project_workspace / 'log.txt', backupCount=5, delay=True)
log_file_handler = RotatingFileHandler(
self.project_workspace / 'log.txt', backupCount=5, delay=True
)
log_file_handler.doRollover()
logging.getLogger('fab').addHandler(log_file_handler)

Expand All @@ -207,9 +226,13 @@ def _init_logging(self):
def _finalise_logging(self):
# remove our file logger
fab_logger = logging.getLogger('fab')
log_file_handlers = list(by_type(fab_logger.handlers, RotatingFileHandler))
log_file_handlers = list(by_type(fab_logger.handlers,
RotatingFileHandler))
if len(log_file_handlers) != 1:
warnings.warn(f'expected to find 1 RotatingFileHandler for removal, found {len(log_file_handlers)}')
warnings.warn(
"expected to find 1 RotatingFileHandler for removal, found "
+ str(len(log_file_handlers))
)
fab_logger.removeHandler(log_file_handlers[0])

def _finalise_metrics(self, start_time, steps_timer):
Expand Down Expand Up @@ -249,14 +272,15 @@ def __init__(self, match: str, flags: List[str]):
# For source in the um folder, add an absolute include path
AddFlags(match="$source/um/*", flags=['-I$source/include']),
# For source in the um folder, add an include path relative to each source file.
# For source in the um folder, add an include path relative to
each source file.
AddFlags(match="$source/um/*", flags=['-I$relative/include']),
"""
self.match: str = match
self.flags: List[str] = flags

# todo: we don't need the project_workspace, we could just pass in the output folder
# todo: we don't need the project_workspace, we could just pass in the
# output folder
def run(self, fpath: Path, input_flags: List[str], config):
"""
Check if our filter matches a given file. If it does, add our flags.
Expand All @@ -269,12 +293,16 @@ def run(self, fpath: Path, input_flags: List[str], config):
Contains the folders for templating `$source` and `$output`.
"""
params = {'relative': fpath.parent, 'source': config.source_root, 'output': config.build_output}
params = {'relative': fpath.parent,
'source': config.source_root,
'output': config.build_output}

# does the file path match our filter?
if not self.match or fnmatch(str(fpath), Template(self.match).substitute(params)):
if not self.match or fnmatch(str(fpath),
Template(self.match).substitute(params)):
# use templating to render any relative paths in our flags
add_flags = [Template(flag).substitute(params) for flag in self.flags]
add_flags = [Template(flag).substitute(params)
for flag in self.flags]

# add our flags
input_flags += add_flags
Expand All @@ -284,16 +312,18 @@ class FlagsConfig():
"""
Return command-line flags for a given path.
Simply allows appending flags but may evolve to also replace and remove flags.
Simply allows appending flags but may evolve to also replace and remove
flags.
"""
def __init__(self, common_flags: Optional[List[str]] = None, path_flags: Optional[List[AddFlags]] = None):
def __init__(self,
common_flags: Optional[List[str]] = None,
path_flags: Optional[List[AddFlags]] = None):
"""
:param common_flags:
List of flags to apply to all files. E.g `['-O2']`.
:param path_flags:
List of :class:`~fab.build_config.AddFlags` objects which apply flags to selected paths.
List of :class:`~fab.build_config.AddFlags` objects which apply
flags to selected paths.
"""
self.common_flags = common_flags or []
self.path_flags = path_flags or []
Expand All @@ -311,8 +341,8 @@ def flags_for_path(self, path: Path, config):
"""
# We COULD make the user pass these template params to the constructor
# but we have a design requirement to minimise the config burden on the user,
# so we take care of it for them here instead.
# but we have a design requirement to minimise the config burden on
# the user, so we take care of it for them here instead.
params = {'source': config.source_root, 'output': config.build_output}
flags = [Template(i).substitute(params) for i in self.common_flags]

Expand Down
Loading

0 comments on commit 1f2f5a5

Please sign in to comment.