Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SofastFixed unit tests and test data #63

Merged
merged 30 commits into from
Apr 2, 2024
Merged

SofastFixed unit tests and test data #63

Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
e292ecf
Renamed example_calculate_dot_locations_from_display_shape_file
braden6521 Mar 29, 2024
fdbf02b
Renamed and moved all sofast-related data into one location, removed …
braden6521 Mar 29, 2024
6b4c8b4
Updated test_CalibrateDisplayShape to use new file locations.
braden6521 Mar 29, 2024
95b18c0
Updated TestDotLocationsFixedPattern to use new file locations.
braden6521 Mar 29, 2024
fa6a6c6
Updated test_image_processing to use new file locations.
braden6521 Mar 29, 2024
f1dce38
test_ImageCalibrationGlobal uses unittest.main()
braden6521 Mar 29, 2024
f6b94ae
test_integration_multi_facet uses new file locations.
braden6521 Mar 29, 2024
edd5055
test_calibrateDisplayShape and test_displayShape save to different di…
braden6521 Mar 29, 2024
aba8fc0
Sofast facet/undefined use new file locations
braden6521 Mar 29, 2024
ff99732
Added Sofast Fixed camera
braden6521 Mar 29, 2024
87cc0c8
test_ProcessSofastFixed uses new file locations.
braden6521 Mar 29, 2024
25c31db
test_project_fixed_pattern_target uses new file locations and unittes…
braden6521 Mar 29, 2024
792d5d4
test_spatial_processing uses new file locations.
braden6521 Mar 29, 2024
a0d84b8
test_spatialOrientation uses new file locations.
braden6521 Mar 29, 2024
19bd83f
test_SystemSofastFringe uses new file locations and unittest.main()
braden6521 Mar 29, 2024
99261f2
test_SystemSofastFixed uses unittest.main()
braden6521 Mar 29, 2024
1a18211
Moved all camera_sofast files to sofast_common and renamed one to cam…
braden6521 Mar 29, 2024
62441a6
updated deflectometry unit tests to use new file locations.
braden6521 Mar 29, 2024
77a2393
Added sofastFixed dot location calibration unit test data
braden6521 Mar 29, 2024
3655249
Fixed bug in CalibrateSofastFixedDots
braden6521 Mar 29, 2024
efa006e
Updated test_CalibrateSofastFixedDots and data downsample script.
braden6521 Mar 29, 2024
87ec271
Added teardown class to testCalibrateSofastFixedDots
braden6521 Mar 29, 2024
00269ed
Updated SofastFringe examples to use new file locations.
braden6521 Mar 29, 2024
ed6cb6b
Renamed sofastFixed example so it does not get run with pytest.
braden6521 Mar 29, 2024
d1ec52c
Made Black happy
braden6521 Mar 29, 2024
3be9679
Updated sofast example inline comments. Made sofast_calibration folder.
braden6521 Apr 1, 2024
879bac7
re-organized sofast test data into input/output folders.
braden6521 Apr 1, 2024
76c6788
Updated unit tests to use new file locations.
braden6521 Apr 1, 2024
aa68056
Updated sofast examples to use new file locations.
braden6521 Apr 1, 2024
5252760
Make black happy.
braden6521 Apr 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 8 additions & 19 deletions contrib/test_data_generation/sofast_fixed/generate_downsampled_calibration_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
"""Generates downsampled dataset used for calibrating the 3d locations of fixed
pattern dots.
"""Generates downsampled dataset for dot_location_calibration
"""

from glob import glob
from os.path import join, basename
import shutil
from os.path import join, basename, abspath
import sys

import imageio.v3 as imageio
Expand All @@ -19,17 +17,14 @@
def generate_data():
"""Downsamples and saves files"""
# Define file locations
dir_sample_data = join(
opencsp_code_dir(), '../../sample_data/deflectometry/calibration_dot_locations/data_measurement'
dir_cal_data = join(
opencsp_code_dir(), '../../sample_data/deflectometry/sandia_lab/dot_locations_calibration/data_measurement'
)

files_images = glob(join(dir_sample_data, 'images/*.JPG'))
file_camera_cal = join(dir_sample_data, 'camera_image_calibration.h5')
file_point_locs = join(dir_sample_data, 'point_locations.csv')
file_camera_def = join(dir_sample_data, 'camera_deflectometry.h5')
file_image_def = join(dir_sample_data, 'image_deflectometry_camera.png')
files_images = glob(abspath(join(dir_cal_data, 'images/*.JPG')))
file_camera_cal = abspath(join(dir_cal_data, 'camera_calibration.h5'))

dir_save = join(opencsp_code_dir(), 'test/data/measurements_sofast_fixed/dot_location_calibration/measurements')
dir_save = join(opencsp_code_dir(), 'test/data/dot_location_calibration')

# Downsample marker/dot images
n_downsample = 4
Expand All @@ -41,19 +36,13 @@ def generate_data():
im_ds = ddg.downsample_images(im, n_downsample)
# Save
file_save = join(dir_save, 'images', basename(file))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI, the "opencsp" way to get the basename is with file_tools.path_components():

file_path, file_name, file_ext = ft.path_components(file)
file_save = join(dir_save, 'images', file_name+file_ext)

This is a little more verbose, and sometimes it is nicer to just use basename. Up to you.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the tip! I like that; in this particular instance, I'll probably keep it as-is because it's simpler, but I'll keep this in mind!

imageio.imwrite(file_save, im_ds, quality=70)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yes, the old struggle. Better images or smaller files?

FYI: because .png and .gif images use a pallet of colors, you can sometimes get smaller files with lossless quality by using one of these formats.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the tip! I think these images might be too high of resolution. I tried png and gif and couldn't get the images smaller than JPG. I'll keep this in mind for the future though!

imageio.imwrite(file_save, im_ds, quality=80)

# Downsample cal camera
print('Downsampling calibration camera...')
cam_cal = ddg.downsample_camera(file_camera_cal, n_downsample)
cam_cal.save_to_hdf(join(dir_save, basename(file_camera_cal)))

# Save other files
shutil.copy(file_point_locs, join(dir_save, basename(file_point_locs)))
shutil.copy(file_camera_def, join(dir_save, basename(file_camera_def)))
shutil.copy(file_image_def, join(dir_save, basename(file_image_def)))


if __name__ == '__main__':
generate_data()
print('Done')
0 ...late_dot_locations_from_display_object.py → ...fixed/dot_locations_from_display_shape.py
View file
Open in desktop
File renamed without changes.
54 changes: 27 additions & 27 deletions example/sofast_fringe/example_calibration_screen_shape.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,30 +13,41 @@
import opencsp.common.lib.tool.log_tools as lt


def run_screen_shape_calibration(save_dir):
def example_screen_shape_calibration():
"""Runs screen shape calibration. Saves a DisplayShape HDF5 file
to ./data/output/screen_shape/display_shape.h5
"""
# Load output data from Scene Reconstruction (Aruco marker xyz points)
file_pts_data = join(
opencsp_code_dir(), 'common/lib/deflectometry/test/data/data_measurement', 'point_locations.csv'
# Define save directory
dir_save = join(dirname(__file__), 'data/output/screen_shape')
ft.create_directories_if_necessary(dir_save)

# Set up logger
lt.logger(join(dir_save, 'log.txt'), lt.log.INFO)

# Define input files
file_pts_data = join(opencsp_code_dir(), 'test/data/sofast_common/aruco_corner_locations.csv')
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should the data be moved to an 'input' subdirectory 'test/data/input/sofast_common/aruco_corner_locations.csv'?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes... I remember thinking this was sloppy and remember thinking that I should totally change it.
Measurement / expected data files are now in representative folders. All unit tests/examples updated.

file_screen_cal_point_pairs = join(
opencsp_code_dir(), 'test/data/display_shape_calibration/data_measurement/screen_calibration_point_pairs.csv'
)
file_camera_distortion = join(
opencsp_code_dir(), 'test/data/display_shape_calibration/data_measurement/camera_screen_shape.h5'
)
file_image_projection = join(opencsp_code_dir(), 'test/data/sofast_common/image_projection.h5')
files_screen_shape_measurement = glob(
join(
opencsp_code_dir(),
'test/data/display_shape_calibration/data_measurement/screen_shape_sofast_measurements/pose_*.h5',
)
)

# Load output data from Scene Reconstruction (Aruco marker xyz points)
pts_marker_data = np.loadtxt(file_pts_data, delimiter=',', skiprows=1)
pts_xyz_marker = Vxyz(pts_marker_data[:, 2:].T)
corner_ids = pts_marker_data[:, 1]

# Define desired resolution of screen sample grid
resolution_xy = [100, 100]

# Define directory where screen shape calibration data is saved
base_dir_sofast_cal = join(opencsp_code_dir(), 'app/sofast/test/data/data_measurement')

# Define input files
file_screen_cal_point_pairs = join(base_dir_sofast_cal, 'screen_calibration_point_pairs.csv')
file_camera_distortion = join(base_dir_sofast_cal, 'camera_screen_shape.h5')
file_image_projection = join(base_dir_sofast_cal, 'image_projection.h5')
files_screen_shape_measurement = glob(join(base_dir_sofast_cal, 'screen_shape_sofast_measurements/pose_*.h5'))

# Load input data
camera = Camera.load_from_hdf(file_camera_distortion)
image_projection_data = ImageProjection.load_from_hdf(file_image_projection)
Expand All @@ -62,27 +73,16 @@ def run_screen_shape_calibration(save_dir):
display_shape = cal.as_DisplayShape('Example display shape')

# Save DisplayShape file
file = join(save_dir, 'display_shape.h5')
file = join(dir_save, 'display_shape.h5')
display_shape.save_to_hdf(file)
lt.info(f'Saved DisplayShape file to {file:s}')

# Save calibration figures
for fig in cal.figures:
file = join(save_dir, fig.get_label() + '.png')
file = join(dir_save, fig.get_label() + '.png')
lt.info(f'Saving figure to: {file:s}')
fig.savefig(file)


def example_driver():
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you mean to delete this method? If yes, then great! I just wanted to make sure you're aware of it.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, since the example_driver() now only directly called example_calibation_screen_shape() I just called that directly.

# Define save directory
save_path = join(dirname(__file__), 'data/output/screen_shape')
ft.create_directories_if_necessary(save_path)

# Set up logger
lt.logger(join(save_path, 'log.txt'), lt.log.INFO)

run_screen_shape_calibration(save_path)


if __name__ == '__main__':
example_driver()
example_screen_shape_calibration()
33 changes: 13 additions & 20 deletions example/sofast_fringe/example_calibration_spatial_orientation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,22 @@
import opencsp.common.lib.tool.log_tools as lt


def run_camera_position_calibration(save_dir):
def example_camera_position_calibration():
"""Calibrates the position of the Sofast camera. Saves the rvec/tvec that
define the relative pose of the camera/screen in a SpatialOrientation file
at ./data/output/spatial_orientation.h5
"""
# Define directory where screen shape calibration data is saved
base_dir_sofast_cal = join(opencsp_code_dir(), 'common/lib/deflectometry/test/data/data_measurement')
# Define save dir
dir_save = join(dirname(__file__), 'data/output/camera_pose')
ft.create_directories_if_necessary(dir_save)

# Set up logger
lt.logger(join(dir_save, 'log.txt'), lt.log.INFO)

# Define inputs
file_camera_sofast = join(base_dir_sofast_cal, 'camera_sofast.h5')
file_cal_image = join(base_dir_sofast_cal, 'image_sofast_camera.png')
file_pts_data = join(base_dir_sofast_cal, 'point_locations.csv')
file_camera_sofast = join(opencsp_code_dir(), 'test/data/sofast_common/camera_sofast.h5')
file_cal_image = join(opencsp_code_dir(), 'test/data/camera_position_calibration/image_sofast_camera.png')
file_pts_data = join(opencsp_code_dir(), 'test/data/sofast_common/aruco_corner_locations.csv')

# Load input data
camera = Camera.load_from_hdf(file_camera_sofast)
Expand Down Expand Up @@ -52,25 +56,14 @@ def run_camera_position_calibration(save_dir):
orientation = SpatialOrientation(r_cam_screen, v_cam_screen_cam)

# Save data
orientation.save_to_hdf(join(save_dir, 'spatial_orientation.h5'))
orientation.save_to_hdf(join(dir_save, 'spatial_orientation.h5'))

# Save figures
for fig in cal.figures:
file = join(save_dir, fig.get_label() + '.png')
file = join(dir_save, fig.get_label() + '.png')
lt.info(f'Saving figure to: {file:s}')
fig.savefig(file)


def example_driver():
# Define save dir
save_path = join(dirname(__file__), 'data/output/camera_pose')
ft.create_directories_if_necessary(save_path)

# Set up logger
lt.logger(join(save_path, 'log.txt'))

run_camera_position_calibration(save_path)


if __name__ == '__main__':
example_driver()
example_camera_position_calibration()
46 changes: 21 additions & 25 deletions example/sofast_fringe/example_process_facet_ensemble.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,34 @@
import opencsp.common.lib.tool.log_tools as lt


def example(dir_save: str):
"""Example SOFAST script
def example_process_facet_ensemble():
"""Example Sofast script

Performs processing of previously collected Sofast data of multi facet mirror ensemble.
1. Loads saved "multi-facet" SOFAST collection data
2. Processes data with SOFAST (without using facet file)
1. Loads saved facet ensemble Sofast collection data
2. Processes data with Sofast
3. Logs best-fit parabolic focal lengths
4. Plots slope magnitude, physical setup
4. Plots slope magnitude
"""
# Define save dir
dir_save = join(dirname(__file__), 'data/output/facet_ensemble')
ft.create_directories_if_necessary(dir_save)

# Set up logger
lt.logger(join(dir_save, 'log.txt'), lt.log.INFO)

# Define sample data directory
sample_data_dir = join(opencsp_code_dir(), 'test/data/measurements_sofast_fringe/')
dir_data_sofast = join(opencsp_code_dir(), 'test/data/sofast_fringe')
dir_data_common = join(opencsp_code_dir(), 'test/data/sofast_common')

# Directory setup
file_measurement = join(sample_data_dir, 'measurement_ensemble.h5')
file_camera = join(sample_data_dir, 'camera.h5')
file_display = join(sample_data_dir, 'display_distorted_2d.h5')
file_orientation = join(sample_data_dir, 'spatial_orientation.h5')
file_calibration = join(sample_data_dir, 'image_calibration.h5')
file_facet = join(sample_data_dir, 'Facet_lab_6x4.json')
file_ensemble = join(sample_data_dir, 'Ensemble_lab_6x4.json')
file_measurement = join(dir_data_sofast, 'measurement_ensemble.h5')
file_camera = join(dir_data_common, 'camera_sofast_downsampled.h5')
file_display = join(dir_data_common, 'display_distorted_2d.h5')
file_orientation = join(dir_data_common, 'spatial_orientation.h5')
file_calibration = join(dir_data_sofast, 'image_calibration.h5')
file_facet = join(dir_data_common, 'Facet_lab_6x4.json')
file_ensemble = join(dir_data_common, 'Ensemble_lab_6x4.json')

# Load data
camera = Camera.load_from_hdf(file_camera)
Expand Down Expand Up @@ -102,16 +109,5 @@ def example(dir_save: str):
sofast.save_to_hdf(f'{dir_save}/data_multifacet.h5')


def example_driver():
# Define save dir
save_path = join(dirname(__file__), 'data/output/facet_ensemble')
ft.create_directories_if_necessary(save_path)

# Set up logger
lt.logger(join(save_path, 'log.txt'), lt.log.INFO)

example(save_path)


if __name__ == '__main__':
example_driver()
example_process_facet_ensemble()
47 changes: 22 additions & 25 deletions example/sofast_fringe/example_process_single_facet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,25 +17,33 @@
import opencsp.common.lib.tool.log_tools as lt


def example(dir_save: str):
"""Example SOFAST script
def example_process_single_facet():
"""Example Sofast script

Performs processing of previously collected Sofast data of single facet mirror.
1. Loads saved "single-facet" SOFAST collection data
2. Processes data with SOFAST
3. Prints best-fit parabolic focal lengths
4. Plots slope magnitude, physical setup
1. Loads saved single facet Sofast collection data
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For particularly long or complicated functions, I also like to describe the steps just like this.

My own personal style is to take this a step further; I add comments in the code with the step numbers. For example:

# 1. Load data
...
# 2. Process
...
# 3. Calculate and log focal length from parabolic fit
...
# 4. Plot slope map

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I like it! That would make the examples much easier to follow! Inline numbers are now added.

2. Processes data with Sofast
3. Logs best-fit parabolic focal lengths
4. Plots slope magnitude
"""
# Define save dir
dir_save = join(dirname(__file__), 'data/output/single_facet')
ft.create_directories_if_necessary(dir_save)

# Set up logger
lt.logger(join(dir_save, 'log.txt'), lt.log.INFO)

# Define sample data directory
sample_data_dir = join(opencsp_code_dir(), 'test/data/measurements_sofast_fringe/')
dir_data_sofast = join(opencsp_code_dir(), 'test/data/sofast_fringe')
dir_data_common = join(opencsp_code_dir(), 'test/data/sofast_common')

# Directory Setup
file_measurement = join(sample_data_dir, 'measurement_facet.h5')
file_camera = join(sample_data_dir, 'camera.h5')
file_display = join(sample_data_dir, 'display_distorted_2d.h5')
file_orientation = join(sample_data_dir, 'spatial_orientation.h5')
file_calibration = join(sample_data_dir, 'image_calibration.h5')
file_facet = join(sample_data_dir, 'Facet_NSTTF.json')
file_measurement = join(dir_data_sofast, 'measurement_facet.h5')
file_camera = join(dir_data_common, 'camera_sofast_downsampled.h5')
file_display = join(dir_data_common, 'display_distorted_2d.h5')
file_orientation = join(dir_data_common, 'spatial_orientation.h5')
file_calibration = join(dir_data_sofast, 'image_calibration.h5')
file_facet = join(dir_data_common, 'Facet_NSTTF.json')

# Load data
camera = Camera.load_from_hdf(file_camera)
Expand Down Expand Up @@ -78,16 +86,5 @@ def example(dir_save: str):
sofast.save_to_hdf(f'{dir_save}/data_singlefacet.h5')


def example_driver():
# Define save dir
save_path = join(dirname(__file__), 'data/output/single_facet')
ft.create_directories_if_necessary(save_path)

# Set up logger
lt.logger(join(save_path, 'log.txt'), lt.log.INFO)

example(save_path)


if __name__ == '__main__':
example_driver()
example_process_single_facet()
35 changes: 16 additions & 19 deletions example/sofast_fringe/example_process_undefined_shape.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import opencsp.common.lib.tool.log_tools as lt


def example(dir_save):
def example_process_undefined_shape_facet():
"""Example SOFAST script

Performs processing of previously collected Sofast data of single facet mirror.
Expand All @@ -25,15 +25,23 @@ def example(dir_save):
3. Prints best-fit parabolic focal lengths
4. Plots slope magnitude, physical setup
"""
# Define save dir
dir_save = join(dirname(__file__), 'data/output/undefined_shape')
ft.create_directories_if_necessary(dir_save)

# Set up logger
lt.logger(join(dir_save, 'log.txt'), lt.log.INFO)

# Define sample data directory
sample_data_dir = join(opencsp_code_dir(), 'test/data/measurements_sofast_fringe/')
dir_data_sofast = join(opencsp_code_dir(), 'test/data/sofast_fringe')
dir_data_common = join(opencsp_code_dir(), 'test/data/sofast_common')

# Directory Setup
file_measurement = join(sample_data_dir, 'measurement_facet.h5')
file_camera = join(sample_data_dir, 'camera.h5')
file_display = join(sample_data_dir, 'display_distorted_2d.h5')
file_orientation = join(sample_data_dir, 'spatial_orientation.h5')
file_calibration = join(sample_data_dir, 'image_calibration.h5')
file_measurement = join(dir_data_sofast, 'measurement_facet.h5')
file_camera = join(dir_data_common, 'camera_sofast_downsampled.h5')
file_display = join(dir_data_common, 'display_distorted_2d.h5')
file_orientation = join(dir_data_common, 'spatial_orientation.h5')
file_calibration = join(dir_data_sofast, 'image_calibration.h5')

# Load data
camera = Camera.load_from_hdf(file_camera)
Expand Down Expand Up @@ -76,16 +84,5 @@ def example(dir_save):
sofast.save_to_hdf(f'{dir_save}/data_undefined.h5')


def example_driver():
# Define save dir
save_path = join(dirname(__file__), 'data/output/undefined_shape')
ft.create_directories_if_necessary(save_path)

# Set up logger
lt.logger(join(save_path, 'log.txt'), lt.log.INFO)

example(save_path)


if __name__ == '__main__':
example_driver()
example_process_undefined_shape_facet()
Loading