From d48637dc9165ca4afcec1ae84efe3430fac2c0ab Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Wed, 7 Aug 2024 16:38:55 -0600 Subject: [PATCH 1/4] example: Setup input and output args for pytest .github/workflows: Added ubi8-weekly.yml --- .github/workflows/ubi8-nightly.yml | 4 +- .github/workflows/ubi8-weekly.yml | 43 +++++++++++++++++++ .github/workflows/ubi8.yml | 2 +- example/conftest.py | 19 ++++++++ .../example_scene_reconstruction.py | 25 ++++++----- 5 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/ubi8-weekly.yml create mode 100644 example/conftest.py diff --git a/.github/workflows/ubi8-nightly.yml b/.github/workflows/ubi8-nightly.yml index a416a536c..29fdc92e0 100644 --- a/.github/workflows/ubi8-nightly.yml +++ b/.github/workflows/ubi8-nightly.yml @@ -33,14 +33,14 @@ jobs: with: path: OpenCSP - - name: pytest-cov + - name: pytest-cov (OpenCSP/example) working-directory: OpenCSP/example run: | python3 -m pip install -r ../requirements.txt export PYTHONPATH=$PWD/../ pytest --color=yes -rs -vv --cov=. --cov-report term --cov-config=.coveragerc - - name: Pip Upgrade pytest-cov + - name: Pip Upgrade pytest-cov (OpenCSP/example) working-directory: OpenCSP/example run: | python3 -m pip install -U -r ../requirements.txt diff --git a/.github/workflows/ubi8-weekly.yml b/.github/workflows/ubi8-weekly.yml new file mode 100644 index 000000000..c82b5f814 --- /dev/null +++ b/.github/workflows/ubi8-weekly.yml @@ -0,0 +1,43 @@ +name: github-UBI8-WEEKLY + +# Runs every Sunday at midnight +on: + workflow_dispatch: + schedule: + - cron: '00 00 * * 0' + +permissions: + contents: none + +# Cancels any in progress 'workflow' associated with this PR +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + ubi8-weekly: + runs-on: [opencsp-latest-ubi8] + permissions: + packages: read + contents: read + + steps: + - name: checkout + uses: actions/checkout@v4 + with: + path: OpenCSP + + - name: pytest-cov (OpenCSP/example - scene_reconstruction) + working-directory: OpenCSP/example + run: | + python3 -m pip install -r ../requirements.txt + export PYTHONPATH=$PWD/../ + pytest \ + --color=yes \ + -rs \ + -vv \ + --cov=. --cov-report term \ + --cov-config=.coveragerc \ + --dir-input=/box_data/scene_reconstruction/data_measurement/ \ + --dir-output=/box_data/scene_reconstruction/data_calculation/ \ + scene_reconstruction/example_scene_reconstruction.py \ No newline at end of file diff --git a/.github/workflows/ubi8.yml b/.github/workflows/ubi8.yml index fce7d68b4..e59ea7b84 100644 --- a/.github/workflows/ubi8.yml +++ b/.github/workflows/ubi8.yml @@ -35,7 +35,7 @@ jobs: with: path: OpenCSP - - name: pytest-cov + - name: pytest-cov (OpenCSP/opencsp) working-directory: OpenCSP/opencsp run: | python3 -m pip install -r ../requirements.txt diff --git a/example/conftest.py b/example/conftest.py new file mode 100644 index 000000000..0dfb0fd92 --- /dev/null +++ b/example/conftest.py @@ -0,0 +1,19 @@ +import pytest + + +# +# Ensure pytest adds root directory to the system path. +# +def pytest_addoption(parser): + parser.addoption('--dir-input', action='store', default='', help='Base directory with data input') + parser.addoption('--dir-output', action='store', default='', help='Base directory where output will be written') + + +@pytest.fixture +def dir_input_fixture(request): + return request.config.getoption('--dir-input') + + +@pytest.fixture +def dir_output_fixture(request): + return request.config.getoption('--dir-output') diff --git a/example/scene_reconstruction/example_scene_reconstruction.py b/example/scene_reconstruction/example_scene_reconstruction.py index cd6fa8c26..33e926aca 100644 --- a/example/scene_reconstruction/example_scene_reconstruction.py +++ b/example/scene_reconstruction/example_scene_reconstruction.py @@ -10,10 +10,8 @@ import opencsp.common.lib.tool.log_tools as lt -def scene_reconstruction(save_dir): +def scene_reconstruction(dir_output, dir_input): """Example script that reconstructs the XYZ locations of Aruco markers in a scene.""" - # Define input directory - dir_input = join(opencsp_code_dir(), 'app/scene_reconstruction/test/data/data_measurement') # Load components camera = Camera.load_from_hdf(join(dir_input, 'camera.h5')) @@ -38,22 +36,29 @@ def scene_reconstruction(save_dir): cal_scene_recon.align_points(marker_ids, alignment_values) # Save points as CSV - cal_scene_recon.save_data_as_csv(join(save_dir, 'point_locations.csv')) + cal_scene_recon.save_data_as_csv(join(dir_output, 'point_locations.csv')) # Save calibrtion figures for fig in cal_scene_recon.figures: - fig.savefig(join(save_dir, fig.get_label() + '.png')) + fig.savefig(join(dir_output, fig.get_label() + '.png')) + +def example_driver(dir_output_fixture, dir_input_fixture): + + dir_input = join(opencsp_code_dir(), 'app/scene_reconstruction/test/data/data_measurement') + dir_output = join(dirname(__file__), 'data/output/scene_reconstruction') + if dir_input_fixture: + dir_input = dir_input_fixture + if dir_output_fixture: + dir_output = dir_input_fixture -def example_driver(): # Define output directory - save_path = join(dirname(__file__), 'data/output/scene_reconstruction') - ft.create_directories_if_necessary(save_path) + ft.create_directories_if_necessary(dir_input) # Set up logger - lt.logger(join(save_path, 'log.txt'), lt.log.INFO) + lt.logger(join(dir_output, 'log.txt'), lt.log.INFO) - scene_reconstruction(save_path) + scene_reconstruction(dir_output, dir_input) if __name__ == '__main__': From d1bd77775a7a50bc270d18df78dc1855a0e18844 Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Thu, 22 Aug 2024 15:48:45 -0600 Subject: [PATCH 2/4] Add docstring --- .../example_scene_reconstruction.py | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/example/scene_reconstruction/example_scene_reconstruction.py b/example/scene_reconstruction/example_scene_reconstruction.py index 33e926aca..78cac0f5f 100644 --- a/example/scene_reconstruction/example_scene_reconstruction.py +++ b/example/scene_reconstruction/example_scene_reconstruction.py @@ -11,7 +11,39 @@ def scene_reconstruction(dir_output, dir_input): - """Example script that reconstructs the XYZ locations of Aruco markers in a scene.""" + """ + Reconstructs the XYZ locations of Aruco markers in a scene. + + Parameters + ---------- + dir_output : str + The directory where the output files, including point locations and calibration figures, will be saved. + dir_input : str + The directory containing the input files needed for scene reconstruction. This includes: + - 'camera.h5': HDF5 file containing camera parameters. + - 'known_point_locations.csv': CSV file with known point locations. + - 'aruco_marker_images/*.JPG': Directory containing images of Aruco markers. + - 'point_pair_distances.csv': CSV file with distances between point pairs. + - 'alignment_points.csv': CSV file with alignment points. + + Notes + ----- + This function performs the following steps: + 1. Loads the camera parameters from an HDF5 file. + 2. Loads known point locations, point pair distances, and alignment points from CSV files. + 3. Initializes the SceneReconstruction object with the camera parameters and known point locations. + 4. Runs the calibration process to determine the marker positions. + 5. Scales the points based on the provided point pair distances. + 6. Aligns the points using the provided alignment points. + 7. Saves the reconstructed point locations to a CSV file. + 8. Saves calibration figures as PNG files in the output directory. + + Examples + -------- + >>> scene_reconstruction('/path/to/output', '/path/to/input') + + """ + # "ChatGPT 4o" assisted with generating this docstring. # Load components camera = Camera.load_from_hdf(join(dir_input, 'camera.h5')) From f97ad56450f3dd5307b52c0064443a6bf2e439af Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Thu, 22 Aug 2024 15:49:22 -0600 Subject: [PATCH 3/4] Include bens feedback --- opencsp/__init__.py | 53 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/opencsp/__init__.py b/opencsp/__init__.py index 2c8c5ba0a..1a0539c3c 100644 --- a/opencsp/__init__.py +++ b/opencsp/__init__.py @@ -15,6 +15,9 @@ import configparser import os +import copy +import sys +import argparse def _opencsp_settings_dirs() -> list[str]: @@ -49,6 +52,55 @@ def _opencsp_settings_dirs() -> list[str]: return ret +def apply_command_line_arguments(settings_from_ini: configparser.ConfigParser) -> configparser.ConfigParser: + settings_mixed = copy.copy(settings_from_ini) + + # parse the command line + parser = argparse.ArgumentParser(prog="OpenCSP/__init__.py", + description='OpenCSP settings parser', add_help=False, exit_on_error=False) + parser.add_argument( + '--dir-input', + dest="dir_input", + default="", + type=str, + help="Use the given directory value as the input directory instead of [opencsp_root_path]/[large_data_example_dir].", + ) + parser.add_argument( + '--dir-output', + dest="dir_output", + default="", + type=str, + help="Use the given directory value as the output directory instead of [opencsp_root_path]/[scratch_dir]/[scratch_name].", + ) + args, remaining = parser.parse_known_args(sys.argv[1:]) + dir_input: str = args.dir_input + dir_output: str = args.dir_output + sys.argv = [sys.argv[0]] + remaining + overridden_values: list[tuple[str, str]] = [] + + # apply the command line arguments to the settings + if dir_input != "": + settings_mixed["opencsp_root_path"]["large_data_example_dir"] = dir_input + overridden_values.append(("opencsp_root_path/large_data_example_dir", dir_input)) + if dir_output != "": + dir_output_path, dir_output_name = os.path.dirname(dir_output), os.path.basename(dir_output) + try: + os.makedirs(dir_output) + except FileExistsError: + pass + settings_mixed["opencsp_root_path"]["scratch_dir"] = dir_output_path + settings_mixed["opencsp_root_path"]["scratch_name"] = dir_output_name + overridden_values.append(("opencsp_root_path/scratch_dir", dir_output_path)) + overridden_values.append(("opencsp_root_path/scratch_name", dir_output_name)) + + # let the user know if values have been overridden + if len(overridden_values) > 0: + print("Some settings have been overridden from the command line:") + for setting_name, command_line_value in overridden_values: + print(f"\t{setting_name}: {command_line_value}") + + return settings_mixed + _settings_files: list[str] = [] @@ -70,4 +122,5 @@ def _opencsp_settings_dirs() -> list[str]: for key in opencsp_settings[section]: print(f"opencsp_settings[{section}][{key}]={opencsp_settings[section][key]}") +opencsp_settings = apply_command_line_arguments(opencsp_settings) __all__ = ['opencsp_settings'] From 06e50ac11edc56867d5fbd7cdd162cd9a2bccd7c Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Mon, 26 Aug 2024 16:47:27 -0600 Subject: [PATCH 4/4] CI fixes --- .../scene_reconstruction/example_scene_reconstruction.py | 2 +- opencsp/__init__.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/example/scene_reconstruction/example_scene_reconstruction.py b/example/scene_reconstruction/example_scene_reconstruction.py index 78cac0f5f..11bc44d37 100644 --- a/example/scene_reconstruction/example_scene_reconstruction.py +++ b/example/scene_reconstruction/example_scene_reconstruction.py @@ -22,7 +22,7 @@ def scene_reconstruction(dir_output, dir_input): The directory containing the input files needed for scene reconstruction. This includes: - 'camera.h5': HDF5 file containing camera parameters. - 'known_point_locations.csv': CSV file with known point locations. - - 'aruco_marker_images/*.JPG': Directory containing images of Aruco markers. + - 'aruco_marker_images/NAME.JPG': Directory containing images of Aruco markers. - 'point_pair_distances.csv': CSV file with distances between point pairs. - 'alignment_points.csv': CSV file with alignment points. diff --git a/opencsp/__init__.py b/opencsp/__init__.py index 1a0539c3c..15b8bd1bb 100644 --- a/opencsp/__init__.py +++ b/opencsp/__init__.py @@ -52,12 +52,14 @@ def _opencsp_settings_dirs() -> list[str]: return ret + def apply_command_line_arguments(settings_from_ini: configparser.ConfigParser) -> configparser.ConfigParser: settings_mixed = copy.copy(settings_from_ini) # parse the command line - parser = argparse.ArgumentParser(prog="OpenCSP/__init__.py", - description='OpenCSP settings parser', add_help=False, exit_on_error=False) + parser = argparse.ArgumentParser( + prog="OpenCSP/__init__.py", description='OpenCSP settings parser', add_help=False, exit_on_error=False + ) parser.add_argument( '--dir-input', dest="dir_input",