From 5a82a72776d0ec2bfa7d2fdaae16d0ebbae0221c Mon Sep 17 00:00:00 2001 From: Neil Shephard Date: Tue, 30 Apr 2024 17:05:29 +0100 Subject: [PATCH 1/4] ci: Adds numpydoc-validation to pre-commit hooks Closes #39 --- .pre-commit-config.yaml | 5 +++++ pyproject.toml | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 352e712..7c06178 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,6 +51,11 @@ repos: hooks: - id: nbstripout + - repo: https://github.com/numpy/numpydoc + rev: v1.6.0 + hooks: + - id: numpydoc-validation + - repo: local hooks: - id: pylint diff --git a/pyproject.toml b/pyproject.toml index 4e10f98..61433ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,4 +141,24 @@ omit = [ "topofileformats/_version.py", "*tests*", "**/__init__*", -] \ No newline at end of file +] + +[tool.numpydoc_validation] +checks = [ + "all", # Perform all check except those listed below + "ES01", + "EX01", + "PR10", # Conflicts with black formatting + "SA01", +] +exclude = [ # don't report on objects that match any of these regex + "\\.undocumented_method$", + "\\.__repr__$", + "^test_", + "^conftest", +] +override_SS05 = [ # override SS05 to allow docstrings starting with these words + "^Process ", + "^Assess ", + "^Access ", +] From 1063ec13e9d44bc9cdfd894f457d7ad879e9de25 Mon Sep 17 00:00:00 2001 From: Neil Shephard Date: Tue, 30 Apr 2024 17:39:33 +0100 Subject: [PATCH 2/4] docs: Applies numpydoc-validation to existing code base. This commit is added to `.git-blame-ignore-revs` so that the blame is not associated with the author. --- .git-blame-ignore-revs | 2 + topofileformats/asd.py | 203 +++++++++++++++++++++++++++-------------- topofileformats/io.py | 81 +++++++++------- 3 files changed, 181 insertions(+), 105 deletions(-) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..afd6bea --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# ns-rse/39-numpydoc-validation : linting docstrings +09c44841ba185de7ed4729fbe6b1d0f58caeb4bc \ No newline at end of file diff --git a/topofileformats/asd.py b/topofileformats/asd.py index 5640b59..59f2001 100644 --- a/topofileformats/asd.py +++ b/topofileformats/asd.py @@ -7,6 +7,7 @@ import numpy as np +import numpy.typing as npt import matplotlib.pyplot as plt from matplotlib import animation @@ -31,14 +32,45 @@ # pylint: disable=too-few-public-methods class VoltageLevelConverter: - """A class for converting arbitrary height levels from the AFM into real world nanometre heights. + """ + A class for converting arbitrary height levels from the AFM into real world nanometre heights. + + Different .asd files require different functions to perform this calculation based on many factors, hence why we + need to define the correct function in each case. - Different .asd files require different functions to perform this calculation - based on many factors, hence why we need to define the correct function in - each case. + Parameters + ---------- + analogue_digital_range : float + The range of analogue voltage values. + max_voltage : float + Maximum voltage. + scaling_factor : float + A scaling factor calculated elsewhere that scales the heightmap appropriately based on the type of channel + and sensor parameters. + resolution : int + The vertical resolution of the instrumen. Dependant on the number of bits used to store its + values. Typically 12, hence 2^12 = 4096 sensitivity levels. """ - def __init__(self, analogue_digital_range, max_voltage, scaling_factor, resolution): + def __init__( + self, analogue_digital_range: float, max_voltage: float, scaling_factor: float, resolution: int + ) -> None: + """ + Convert arbitrary height levels from the AFM into real world nanometre heights. + + Parameters + ---------- + analogue_digital_range : float + The range of analogue voltage values. + max_voltage : float + Maximum voltage. + scaling_factor : float + A scaling factor calculated elsewhere that scales the heightmap appropriately based on the type of channel + and sensor parameters. + resolution : int + The vertical resolution of the instrumen. Dependant on the number of bits used to store its + values. Typically 12, hence 2^12 = 4096 sensitivity levels. + """ self.ad_range = int(analogue_digital_range, 16) self.max_voltage = max_voltage self.scaling_factor = scaling_factor @@ -53,12 +85,13 @@ def __init__(self, analogue_digital_range, max_voltage, scaling_factor, resoluti class UnipolarConverter(VoltageLevelConverter): """A VoltageLevelConverter for unipolar encodings. (0 to +X Volts).""" - def level_to_voltage(self, level): - """Calculate the real world height scale in nanometres for an arbitrary level value. + def level_to_voltage(self, level: float) -> float: + """ + Calculate the real world height scale in nanometres for an arbitrary level value. Parameters ---------- - level: float + level : float Arbitrary height measurement from the AFM that needs converting into real world length scale units. @@ -74,12 +107,13 @@ def level_to_voltage(self, level): class BipolarConverter(VoltageLevelConverter): """A VoltageLevelConverter for bipolar encodings. (-X to +X Volts).""" - def level_to_voltage(self, level): - """Calculate the real world height scale in nanometres for an arbitrary level value. + def level_to_voltage(self, level: float) -> float: + """ + Calculate the real world height scale in nanometres for an arbitrary level value. Parameters ---------- - level: float + level : float Arbitrary height measurement from the AFM that needs converting into real world length scale units. @@ -98,27 +132,28 @@ def calculate_scaling_factor( scanner_sensitivity: float, phase_sensitivity: float, ) -> float: - """Calculate the correct scaling factor. + """ + Calculate the correct scaling factor. This function should be used in conjunction with the VoltageLevelConverter class to define the correct function and enables conversion between arbitrary level values from the AFM into real world nanometre height values. Parameters ---------- - channel: str + channel : str The .asd channel being used. - z_piezo_gain: float + z_piezo_gain : float The z_piezo_gain listed in the header metadata for the .asd file. - z_piezo_extension: float + z_piezo_extension : float The z_piezo_extension listed in the header metadata for the .asd file. - scanner_sensitivity: float + scanner_sensitivity : float The scanner_sensitivity listed in the header metadata for the .asd file. - phase_sensitivity: float + phase_sensitivity : float The phase_sensitivity listed in the heder metadata for the .asd file. Returns ------- - scaling_factor: float + float The appropriate scaling factor to pass to a VoltageLevelConverter to convert arbitrary height levels to real world nanometre heights for the frame data in the specified channl in the .asd file. @@ -146,29 +181,30 @@ def calculate_scaling_factor( def load_asd(file_path: Path, channel: str): - """Load a .asd file. + """ + Load a .asd file. Parameters ---------- - file_path: Path + file_path : Path Path to the .asd file. - channel: str + channel : str Channel to load. Note that only three channels seem to be present in a single .asd file. Options: TP - (Topograph), ER (Error) and PH (Phase). + (Topograph), ER (Error) and PH (Phase). Returns ------- - frames: np.ndarray + npt.NDArray The .asd file frames data as a numpy 3D array N x W x H (Number of frames x Width of each frame x height of each frame). - pixel_to_nanometre_scaling_factor: float + float The number of nanometres per pixel for the .asd file. (AKA the resolution). Enables converting between pixels and nanometres when working with the data, in order to use real-world length - scales. - metadata: dict + scales. + dict Metadata for the .asd file. The number of entries is too long to list here, and changes based on the file - version please either look into the `read_header_file_version_x` functions or print the keys too see what metadata - is available. + version please either look into the `read_header_file_version_x` functions or print the keys too see what + metadata is available. """ # Binary mode open does not take an encoding argument # pylint: disable=unspecified-encoding @@ -243,15 +279,15 @@ def load_asd(file_path: Path, channel: str): return frames, pixel_to_nanometre_scaling_factor, header_dict -def read_file_version(open_file): - """Read the file version from an open asd file. File versions are 0, 1 and 2. +def read_file_version(open_file: BinaryIO) -> int: + """ + Read the file version from an open asd file. File versions are 0, 1 and 2. - Different file versions require different functions to read the headers as - the formatting changes between them. + Different file versions require different functions to read the headers as the formatting changes between them. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object for a .asd file. Returns @@ -265,17 +301,18 @@ def read_file_version(open_file): # pylint: disable=too-many-statements -def read_header_file_version_0(open_file: BinaryIO): - """Read the header metadata for a .asd file using file version 0. +def read_header_file_version_0(open_file: BinaryIO) -> dict: + """ + Read the header metadata for a .asd file using file version 0. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object for a .asd file. Returns ------- - header_dict: dict + dict Dictionary of metadata decoded from the file header. """ header_dict = {} @@ -366,16 +403,17 @@ def read_header_file_version_0(open_file: BinaryIO): # pylint: disable=too-many-statements def read_header_file_version_1(open_file: BinaryIO): - """Read the header metadata for a .asd file using file version 1. + """ + Read the header metadata for a .asd file using file version 1. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object for a .asd file. Returns ------- - header_dict: dict + dict Dictionary of metadata decoded from the file header. """ header_dict = {} @@ -470,17 +508,18 @@ def read_header_file_version_1(open_file: BinaryIO): # pylint: disable=too-many-statements -def read_header_file_version_2(open_file): - """Read the header metadata for a .asd file using file version 2. +def read_header_file_version_2(open_file: BinaryIO) -> dict: + """ + Read the header metadata for a .asd file using file version 2. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object for a .asd file. Returns ------- - header_dict: dict + dict Dictionary of metadata decoded from the file header. """ header_dict = {} @@ -607,25 +646,32 @@ def read_header_file_version_2(open_file): return header_dict -def read_channel_data(open_file, num_frames, x_pixels, y_pixels, analogue_digital_converter): - """Read frame data from an open .asd file, starting at the current position. +def read_channel_data( + open_file: BinaryIO, + num_frames: int, + x_pixels: int, + y_pixels: int, + analogue_digital_converter: VoltageLevelConverter, +) -> npt.NDArray: + """ + Read frame data from an open .asd file, starting at the current position. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object for a .asd file. - num_frames: int + num_frames : int The number of frames for this set of frame data. - x_pixels: int + x_pixels : int The width of each frame in pixels. - y_pixels: int + y_pixels : int The height of each frame in pixels. - analogue_digital_converter: A VoltageLevelConverter instance for converting the raw - level values to real world nanometre vertical heights. + analogue_digital_converter : VoltageLevelConverter + A VoltageLevelConverter instance for converting the raw level values to real world nanometre vertical heights. Returns ------- - frames: np.ndarray + np.ndarray The extracted frame heightmap data as a N x W x H 3D numpy array (number of frames x width of each frame x height of each frame). Units are nanometres. """ @@ -665,25 +711,28 @@ def read_channel_data(open_file, num_frames, x_pixels, y_pixels, analogue_digita return frames -def create_analogue_digital_converter(analogue_digital_range, scaling_factor, resolution=4096): - """Create an analogue to digital converter for a given range, scaling factor and resolution. +def create_analogue_digital_converter( + analogue_digital_range: float, scaling_factor: float, resolution: int = 4096 +) -> VoltageLevelConverter: + """ + Create an analogue to digital converter for a given range, scaling factor and resolution. Used for converting raw level values into real world height scales in nanometres. Parameters ---------- - analogue_digital_range: float + analogue_digital_range : float The range of analogue voltage values. - scaling factor: float - A scaling factor calculated elsewhere that scales the heightmap appropriately based on - the type of channel and sensor parameters. - resolution: int - The vertical resolution of the instrumen. Dependant on the number of bits used to store - its values. Typically 12, hence 2^12 = 4096 sensitivity levels. + scaling_factor : float + A scaling factor calculated elsewhere that scales the heightmap appropriately based on the type of channel and + sensor parameters. + resolution : int + The vertical resolution of the instrumen. Dependant on the number of bits used to store its values. Typically + 12, hence 2^12 = 4096 sensitivity levels. Returns ------- - converter: VoltageLevelConverter + VoltageLevelConverter An instance of the VoltageLevelConverter class with a tailored function `level_to_voltage` which converts arbitrary level values into real world nanometre heights for the given .asd file. Note that this is file specific since the parameters will change between files. @@ -759,24 +808,38 @@ def create_analogue_digital_converter(analogue_digital_range, scaling_factor, re return converter -def create_animation(file_name: str, frames: np.ndarray, file_format: str = ".gif") -> None: - """Create animation from a numpy array of frames (2d numpy arrays). +def create_animation(file_name: str, frames: npt.NDArray, file_format: str = ".gif") -> None: + """ + Create animation from a numpy array of frames (2d numpy arrays). File format can be specified, defaults to .gif. Parameters ---------- - file_name: str - Name of the file to save - frames: np.ndarray + file_name : str + Name of the file to save. + frames : npt.NDArray Numpy array of frames of shape (N x W x H) where N is the number of frames, W is the width of the frames and H is the height of the frames. - file_format: str + file_format : str Optional string for the file format to save as. Formats currently available: .mp4, .gif. """ fig, axis = plt.subplots() - def update(frame): + def update(frame: npt.NDArray): + """ + Update the image with the latest frame. + + Parameters + ---------- + frame : npt.NDArray + Single frame to add to the image. + + Returns + ------- + axis + Matplotlib axis. + """ axis.imshow(frames[frame]) return axis diff --git a/topofileformats/io.py b/topofileformats/io.py index 1dd5e02..4189626 100644 --- a/topofileformats/io.py +++ b/topofileformats/io.py @@ -5,11 +5,12 @@ def read_uint8(open_file: BinaryIO) -> int: - """Read an unsigned 8 bit integer from an open binary file. + """ + Read an unsigned 8 bit integer from an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. Returns @@ -21,11 +22,12 @@ def read_uint8(open_file: BinaryIO) -> int: def read_int8(open_file: BinaryIO) -> int: - """Read a signed 8 bit integer from an open binary file. + """ + Read a signed 8 bit integer from an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. Returns @@ -37,11 +39,12 @@ def read_int8(open_file: BinaryIO) -> int: def read_int16(open_file: BinaryIO) -> int: - """Read a signed 16 bit integer from an open binary file. + """ + Read a signed 16 bit integer from an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. Returns @@ -53,11 +56,12 @@ def read_int16(open_file: BinaryIO) -> int: def read_int32(open_file: BinaryIO) -> int: - """Read a signed 32 bit integer from an open binary file. + """ + Read a signed 32 bit integer from an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. Returns @@ -69,11 +73,12 @@ def read_int32(open_file: BinaryIO) -> int: def read_uint32(open_file: BinaryIO) -> int: - """Read an unsigned 32 bit integer from an open binary file. + """ + Read an unsigned 32 bit integer from an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. Returns @@ -85,11 +90,12 @@ def read_uint32(open_file: BinaryIO) -> int: def read_hex_u32(open_file: BinaryIO) -> str: - """Read a hex encoded unsigned 32 bit integer value from an open binary file. + """ + Read a hex encoded unsigned 32 bit integer value from an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. Returns @@ -101,101 +107,106 @@ def read_hex_u32(open_file: BinaryIO) -> str: def read_float(open_file: BinaryIO) -> float: - """Read a float from an open binary file. + """ + Read a float from an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. Returns ------- float - Float decoded value. + Float decoded value. """ return struct.unpack("f", open_file.read(4))[0] def read_bool(open_file: BinaryIO) -> bool: - """Read a boolean from an open binary file. + """ + Read a boolean from an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. Returns ------- bool - Boolean decoded value + Boolean decoded value. """ return bool(int.from_bytes(open_file.read(1), byteorder="little")) def read_double(open_file: BinaryIO) -> float: - """Read an 8 byte double from an open binary file. + """ + Read an 8 byte double from an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. Returns ------- float - Float decoded from the double value + Float decoded from the double value. """ return struct.unpack("d", open_file.read(8))[0] def read_ascii(open_file: BinaryIO, length_bytes: int = 1) -> str: - """Read an ascii string of defined length from an open binary file. + """ + Read an ASCII string of defined length from an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. - length_bytes: int - Length of the ascii string in bytes that should be read. Default: 1 byte (1 character). + length_bytes : int + Length of the ASCII string in bytes that should be read. Default: 1 byte (1 character). Returns ------- str - Ascii text decoded from file. + ASCII text decoded from file. """ return open_file.read(length_bytes).decode("ascii") def read_null_separated_utf8(open_file: BinaryIO, length_bytes: int = 2) -> str: - r"""Read an ASCII string of defined length from an open binary file. + r""" + Read an ASCII string of defined length from an open binary file. Each character is separated by a null byte. This encoding is known as UTF-16LE (Little Endian). Eg: b'\x74\x00\x6f\x00\x70\x00\x6f' would decode to 'topo' in this format. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. - length_bytes: int - Length of the ascii string in bytes that should be read. Default: 2 bytes - (1 UTF-16LE character) + length_bytes : int + Length of the ASCII string in bytes that should be read. Default: 2 bytes (1 UTF-16LE character). Returns ------- str - Ascii text decoded from file. + ASCII text decoded from file. """ return open_file.read(length_bytes).replace(b"\x00", b"").decode("ascii") def skip_bytes(open_file: BinaryIO, length_bytes: int = 1) -> bytes: - """Skip a specified number of bytes when reading an open binary file. + """ + Skip a specified number of bytes when reading an open binary file. Parameters ---------- - open_file: BinaryIO + open_file : BinaryIO An open binary file object. - length_bytes: int + length_bytes : int Number of bytes to skip. Returns From 3dbfd72590f55c134fceff88486141c2da31e811 Mon Sep 17 00:00:00 2001 From: Neil Shephard Date: Tue, 30 Apr 2024 17:41:51 +0100 Subject: [PATCH 3/4] chore: Tidying up configuration of ruff --- pyproject.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 61433ae..152f623 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,26 +113,26 @@ exclude = [ # per-file-ignores = [] line-length = 120 target-version = "py310" -select = ["A", "B", "C", "D", "E", "F", "PT", "PTH", "R", "S", "W", "U"] -ignore = [ +lint.select = ["A", "B", "C", "D", "E", "F", "PT", "PTH", "R", "S", "W", "U"] +lint.ignore = [ "B905", "E501", "S101", "T201"] # Allow autofix for all enabled rules (when `--fix`) is provided. -fixable = ["A", "B", "C", "D", "E", "F", "PT", "PTH", "R", "S", "W", "U"] -unfixable = [] +lint.fixable = ["A", "B", "C", "D", "E", "F", "PT", "PTH", "R", "S", "W", "U"] +lint.unfixable = [] -[tool.ruff.flake8-quotes] +[tool.ruff.lint.flake8-quotes] docstring-quotes = "double" -[tool.ruff.isort] +[tool.ruff.lint.isort] case-sensitive = true -[tool.ruff.pydocstyle] +[tool.ruff.lint.pydocstyle] convention = "numpy" -[tool.ruff.flake8-pytest-style] +[tool.ruff.lint.flake8-pytest-style] fixture-parentheses = true [tool.coverage.run] From 3c045c41b4b4cbc3d247836df924187318a59c58 Mon Sep 17 00:00:00 2001 From: Neil Shephard Date: Wed, 1 May 2024 12:05:12 +0100 Subject: [PATCH 4/4] Update topofileformats/asd.py Co-authored-by: Sylvia Whittle <86117496+SylviaWhittle@users.noreply.github.com> --- topofileformats/asd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/topofileformats/asd.py b/topofileformats/asd.py index 59f2001..4c31dc4 100644 --- a/topofileformats/asd.py +++ b/topofileformats/asd.py @@ -48,7 +48,7 @@ class VoltageLevelConverter: A scaling factor calculated elsewhere that scales the heightmap appropriately based on the type of channel and sensor parameters. resolution : int - The vertical resolution of the instrumen. Dependant on the number of bits used to store its + The vertical resolution of the instrument. Dependant on the number of bits used to store its values. Typically 12, hence 2^12 = 4096 sensitivity levels. """