From 2f338ffec915f3a0376f682db35d1874bcae948f Mon Sep 17 00:00:00 2001 From: bbean Date: Fri, 15 Mar 2024 10:12:13 -0600 Subject: [PATCH 01/12] ignore all files in __pycache__ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ca0f50d21..a6a224742 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.pyc .pytest_cache/ .spyproject +**/__pycache__/ # Virtual environment env/ From a291ec6875c75043588bc6ff8f59ea193636da39 Mon Sep 17 00:00:00 2001 From: bbean Date: Fri, 15 Mar 2024 10:12:31 -0600 Subject: [PATCH 02/12] fix test_sensitive_strings.py --- contrib/scripts/test/test_sensitive_strings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/scripts/test/test_sensitive_strings.py b/contrib/scripts/test/test_sensitive_strings.py index 7078445c7..7ad5f18d5 100644 --- a/contrib/scripts/test/test_sensitive_strings.py +++ b/contrib/scripts/test/test_sensitive_strings.py @@ -61,7 +61,7 @@ def test_all_matches(self): sensitive_strings_csv = os.path.join(self.ss_dir, "test_all_matches.csv") searcher = ss.SensitiveStringsSearcher(self.root_search_dir, sensitive_strings_csv, - self.all_binaries) + self.no_binaries) searcher.git_files_only = False # 6 matches: # files: a.txt, b/b.txt, c/d/e.txt From b76597b3dc0f0b6baf3ae5040ce283e462856301 Mon Sep 17 00:00:00 2001 From: bbean Date: Fri, 15 Mar 2024 10:30:29 -0600 Subject: [PATCH 03/12] fix test_opencsp_root_path.py --- .../common/lib/opencsp_path/test/test_opencsp_root_path.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencsp/common/lib/opencsp_path/test/test_opencsp_root_path.py b/opencsp/common/lib/opencsp_path/test/test_opencsp_root_path.py index 7c2d93d43..6dd09a768 100644 --- a/opencsp/common/lib/opencsp_path/test/test_opencsp_root_path.py +++ b/opencsp/common/lib/opencsp_path/test/test_opencsp_root_path.py @@ -47,8 +47,8 @@ def get_opencsp_path(self): def test_opencsp_code_dir(self): opencsp_path = self.get_opencsp_path() - expected = os.path.normpath(opencsp_path) - actual = os.path.normpath(orp.opencsp_code_dir()) + expected = os.path.normpath(opencsp_path).lower() + actual = os.path.normpath(orp.opencsp_code_dir()).lower() self.assertEqual(expected, actual) def test_opencsp_doc_dir(self): From affeef1828b40ed34f655481e08d6c169e1babb2 Mon Sep 17 00:00:00 2001 From: bbean Date: Fri, 15 Mar 2024 10:48:15 -0600 Subject: [PATCH 04/12] fix test_ImageAttributeParser.py --- .../lib/render/test/test_ImageAttributeParser.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/opencsp/common/lib/render/test/test_ImageAttributeParser.py b/opencsp/common/lib/render/test/test_ImageAttributeParser.py index 1cb259dd9..dbcf58af7 100644 --- a/opencsp/common/lib/render/test/test_ImageAttributeParser.py +++ b/opencsp/common/lib/render/test/test_ImageAttributeParser.py @@ -16,12 +16,20 @@ def setUp(self) -> None: self.img_file = os.path.join( self.out_dir, f"nonexistant_image_{self._testMethodName}.png" ) - self.attr_file = os.path.join( + attr_file_src = os.path.join( self.data_dir, f"nonexistant_image_{self._testMethodName}.txt" ) + self.attr_file = os.path.join( + self.out_dir, f"nonexistant_image_{self._testMethodName}.txt" + ) ft.create_directories_if_necessary(self.out_dir) + # Some tests have attribute files that need to be copied to self.out_dir. + # Copy the attribute files, if they exist. + if not ft.file_exists(self.attr_file) and ft.file_exists(attr_file_src): + ft.copy_file(attr_file_src, self.out_dir) + def test_no_attrfile(self): """Contructor succeeds even without attributes file""" parser = iap.ImageAttributeParser(self.img_file) From 0b6fdcff3c614df5a18e5d409c11472029a7bf86 Mon Sep 17 00:00:00 2001 From: bbean Date: Fri, 15 Mar 2024 13:44:47 -0600 Subject: [PATCH 05/12] fix test_photogrammetry.py --- requirements.txt | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/requirements.txt b/requirements.txt index 79c585c19..8f78c48d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,30 +9,30 @@ # Note, please leave the hashtags (eg #tag) in place. These are here for processing this file for parallel execution. # packages that can be installed with conda on solo #condapkgs -h5py >= 3.7.0 -imageio >= 2.19.3 -matplotlib >= 3.6.2 -numpy >= 1.23.5 -pandas >= 1.5.2 -pillow >= 9.3.0 -pyproj >= 3.4.1 -pytz >= 2022.7 -scipy >= 1.10.0 -sympy >= 1.11.1 +h5py >= 3.10.0 +imageio >= 2.34.0 +matplotlib >= 3.8.3 +numpy >= 1.26.4 +pandas >= 2.2.1 +pillow >= 10.2.0 +pyproj >= 3.6.1 +pytz >= 2024.1 +scipy >= 1.12.0 +sympy >= 1.12 # packages that can't be installed with conda but can be installed with pip on solo #pippkgs -ipykernel >= 6.22.0 +ipykernel >= 6.29.3 opencv-contrib-python == 4.5.5.64 -pytest >= 7.2.0 -python-pptx >= 0.6.21 -rawpy >= 0.18.1 +pytest >= 8.1.1 +python-pptx >= 0.6.23 +rawpy >= 0.19.1 openpyxl >= 3.1.2 # these are packages that have errors with both conda and pip on solo #errpkgs -pypylon >= 1.9.0 +pypylon >= 3.0.1 pysolar >= 0.11 -tqdm >= 4.66.1 +tqdm >= 4.66.2 -pytest-xvfb -pytest-cov \ No newline at end of file +pytest-xvfb >= 3.0.0 +pytest-cov >= 4.1.0 \ No newline at end of file From 4c0cc40aaf996d17c74d42e365500fd560e15b46 Mon Sep 17 00:00:00 2001 From: bbean Date: Fri, 15 Mar 2024 17:10:48 -0600 Subject: [PATCH 06/12] fix test_figure_management.py --- .../common/lib/render/figure_management.py | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/opencsp/common/lib/render/figure_management.py b/opencsp/common/lib/render/figure_management.py index de76f278c..68a35e598 100644 --- a/opencsp/common/lib/render/figure_management.py +++ b/opencsp/common/lib/render/figure_management.py @@ -57,6 +57,36 @@ def reset_figure_management(): fig_record_list = [] +def _mpl_pyplot_figure(*vargs, **kwargs): + """ Initializes and returns a matplotlib.pyplot.figure() instance. + + If creating the figure fails, try again (up to two more times). + + Sometimes initializing a matplotlib figure fails. But if you try to + initialize the figure() class again, it seems to always succeed. The + measured failure rate for first time initialization is between 7% and 15%. + When it fails, it is often with an error about not being able to find a + file. Something like:: + + _tkinter.TclError: Can't find a usable init.tcl in the following directories + """ + try: + # try to create a figure + return plt.figure(*vargs, **kwargs) + except Exception: + try: + lt.warn("Failed to create a matplotlib.pyplot.figure instance. Trying again (2nd attempt).") + # first attempt failed, try again + return plt.figure(*vargs, **kwargs) + except Exception: + # second attempt failed, give the system a second to stabalize and + # try a third time + lt.warn("Failed to create a matplotlib.pyplot.figure instance. Trying again (3rd attempt).") + import time + time.sleep(1) + return plt.figure(*vargs, **kwargs) + + def tile_figure( name=None, # Handle and title of figure window. tile_array: tuple[int, int] = (3, 2), # (n_y, n_x) ~ (columns, rows) @@ -104,8 +134,8 @@ def tile_figure( ul_y = size_y_pixels * y_idx # Create figure. - # fig = plt.figure(constrained_layout=True).subplots(5, 5) - fig = plt.figure(name, figsize=(size_x, plot_size_y)) + # fig = figure(constrained_layout=True).subplots(5, 5) + fig = _mpl_pyplot_figure(name, figsize=(size_x, plot_size_y)) # Turn off the axis around the plot drawing area. This leads to confusing duplicate, mismatched, # axis information. Why this suddenly appeared is beyond me. - RCB # The command below does not suppress the actual plot axes. @@ -177,7 +207,8 @@ def _setup_figure( comments: list[ str ] = None, # List of strings including comments to associate with the figure. - code_tag: str = None, # String of form "code_file.function_name()" showing where to look in code for call that generated this figure. + # String of form "code_file.function_name()" showing where to look in code for call that generated this figure. + code_tag: str = None, ) -> RenderControlFigureRecord: """Common figure setup for 2D and 3D data.""" # defaults @@ -211,7 +242,7 @@ def _setup_figure( tile_square=figure_control.tile_square, ) else: - fig = plt.figure(name, figsize=figure_control.figsize) + fig = _mpl_pyplot_figure(name, figsize=figure_control.figsize) if figure_control.upper_left_xy: upper_left_xy = figure_control.upper_left_xy x = upper_left_xy[0] @@ -465,7 +496,7 @@ def display_plot( if tile: fig = tile_figure(name, tile_array=tile_array) else: - fig = plt.figure(name, figsize=figsize) + fig = _mpl_pyplot_figure(name, figsize=figsize) if upper_left_xy: x = upper_left_xy[0] y = upper_left_xy[1] @@ -498,7 +529,7 @@ def display_bar( if tile: fig = tile_figure(name, tile_array=tile_array) else: - fig = plt.figure(name, figsize=figsize) + fig = _mpl_pyplot_figure(name, figsize=figsize) if upper_left_xy: x = upper_left_xy[0] y = upper_left_xy[1] From 35fb4938964b6fdb5405c686b336d58cc0471ee4 Mon Sep 17 00:00:00 2001 From: bbean Date: Fri, 15 Mar 2024 17:11:22 -0600 Subject: [PATCH 07/12] avoid tripping deprecation message from logging.warn --- opencsp/common/lib/tool/log_tools.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/opencsp/common/lib/tool/log_tools.py b/opencsp/common/lib/tool/log_tools.py index a8bf583ce..2e57df9af 100644 --- a/opencsp/common/lib/tool/log_tools.py +++ b/opencsp/common/lib/tool/log_tools.py @@ -215,7 +215,11 @@ def info(*vargs, **kwargs) -> int: return 0 -def warn(*vargs, **kwargs) -> int: +def warn(*vargs, **kwargs): + warning(*vargs, **kwargs) + + +def warning(*vargs, **kwargs): """Warning message, both to console and log file. Use this level as an indication that something unexpected happened, or From fdc3054bc7b12e44f34e3dd4cd8fda48ee25d55b Mon Sep 17 00:00:00 2001 From: bbean Date: Fri, 15 Mar 2024 17:14:10 -0600 Subject: [PATCH 08/12] incorporate Braden's suggested improvement to photogrammetry.py --- opencsp/common/lib/photogrammetry/photogrammetry.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/opencsp/common/lib/photogrammetry/photogrammetry.py b/opencsp/common/lib/photogrammetry/photogrammetry.py index 752e0a724..cb2d5ddfa 100644 --- a/opencsp/common/lib/photogrammetry/photogrammetry.py +++ b/opencsp/common/lib/photogrammetry/photogrammetry.py @@ -282,7 +282,10 @@ def align_merit_fcn(vec: ndarray): return np.sqrt(np.mean(e**2)) # Optimize points - vec = np.array([0, 0, 0, 0, 0, 0, 1], dtype=float) + if scale: + vec = np.array([0, 0, 0, 0, 0, 0, 1], dtype=float) + else: + vec = np.array([0, 0, 0, 0, 0, 0], dtype=float) x = minimize(align_merit_fcn, vec, method='Powell') # Calculate final alignment error @@ -291,7 +294,8 @@ def align_merit_fcn(vec: ndarray): # Return transform and scale rot_out = Rotation.from_rotvec(x.x[:3]) trans_out = Vxyz(x.x[3:6]) - return TransformXYZ.from_R_V(rot_out, trans_out), x.x[6], e_final + scale_out = 1.0 if not scale else x.x[6] + return TransformXYZ.from_R_V(rot_out, trans_out), scale_out, e_final def _ref_coord_error(pts_obj: Vxyz, pts_exp: Vxyz) -> np.ndarray: From 54ded71138ce4767879d9d4a8a687a216c78c768 Mon Sep 17 00:00:00 2001 From: bbean Date: Tue, 19 Mar 2024 16:05:40 -0600 Subject: [PATCH 09/12] don't include ufacet tests --- ...NAL_SCRATCH.py => NoTest_Heliostat3dInfer_ORIGINAL_SCRATCH.py} | 0 .../test/{Test_Heliostat3dInfer.py => NoTest_Heliostat3dInfer.py} | 0 .../test/{Test_Reconstruct.py => NoTest_Reconstruct.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename contrib/app/ufacet-s/helio_scan/lib/{Test_Heliostat3dInfer_ORIGINAL_SCRATCH.py => NoTest_Heliostat3dInfer_ORIGINAL_SCRATCH.py} (100%) rename contrib/app/ufacet-s/helio_scan/test/{Test_Heliostat3dInfer.py => NoTest_Heliostat3dInfer.py} (100%) rename contrib/app/ufacet-s/helio_scan/test/{Test_Reconstruct.py => NoTest_Reconstruct.py} (100%) diff --git a/contrib/app/ufacet-s/helio_scan/lib/Test_Heliostat3dInfer_ORIGINAL_SCRATCH.py b/contrib/app/ufacet-s/helio_scan/lib/NoTest_Heliostat3dInfer_ORIGINAL_SCRATCH.py similarity index 100% rename from contrib/app/ufacet-s/helio_scan/lib/Test_Heliostat3dInfer_ORIGINAL_SCRATCH.py rename to contrib/app/ufacet-s/helio_scan/lib/NoTest_Heliostat3dInfer_ORIGINAL_SCRATCH.py diff --git a/contrib/app/ufacet-s/helio_scan/test/Test_Heliostat3dInfer.py b/contrib/app/ufacet-s/helio_scan/test/NoTest_Heliostat3dInfer.py similarity index 100% rename from contrib/app/ufacet-s/helio_scan/test/Test_Heliostat3dInfer.py rename to contrib/app/ufacet-s/helio_scan/test/NoTest_Heliostat3dInfer.py diff --git a/contrib/app/ufacet-s/helio_scan/test/Test_Reconstruct.py b/contrib/app/ufacet-s/helio_scan/test/NoTest_Reconstruct.py similarity index 100% rename from contrib/app/ufacet-s/helio_scan/test/Test_Reconstruct.py rename to contrib/app/ufacet-s/helio_scan/test/NoTest_Reconstruct.py From 10a340386716eb76a0c4fd2c5c512be0049c505b Mon Sep 17 00:00:00 2001 From: bbean Date: Tue, 19 Mar 2024 16:06:42 -0600 Subject: [PATCH 10/12] added options to accept all changes when you're confident that all binary file changes are from renames and the contents haven't actually changed (note: still does string matching verification against these files and their names) --- contrib/scripts/sensitive_strings.py | 165 +++++++++++++++++---------- 1 file changed, 102 insertions(+), 63 deletions(-) diff --git a/contrib/scripts/sensitive_strings.py b/contrib/scripts/sensitive_strings.py index 1da9e2ba6..be287994f 100644 --- a/contrib/scripts/sensitive_strings.py +++ b/contrib/scripts/sensitive_strings.py @@ -39,13 +39,16 @@ def __init__( self.sensitive_strings_csv = sensitive_strings_csv self.allowed_binary_files_csv = allowed_binary_files_csv self.cache_file_csv = cache_file_csv - self.interactive = False + self._interactive = False + self.verify_all_on_behalf_of_user = False + self.remove_unfound_binaries = False self.date_time_str = tdt.current_date_time_string_forfile() self.tmp_dir_base = ft.norm_path( os.path.join(orp.opencsp_temporary_dir(), "SensitiveStringSearcher") ) self.git_files_only = True self.is_hdf5_searcher = False + self.has_backed_up_allowed_binaries_csv = False self.matchers = self.build_matchers() self.matches: dict[str, list[ssm.Match]] = {} @@ -56,6 +59,14 @@ def __init__( self.cached_cleared_files: list[fc.FileCache] = [] self.new_cached_cleared_files: list[fc.FileCache] = [] + @property + def interactive(self): + return self._interactive or self.verify_all_on_behalf_of_user + + @interactive.setter + def interactive(self, val: bool): + self._interactive = val + def __del__(self): if ft.directory_exists(self.tmp_dir_base): tmp_dirs = ft.files_in_directory(self.tmp_dir_base, files_only=False) @@ -187,6 +198,7 @@ def search_hdf5_file(self, hdf5_file: ff.FileFingerprint): h5_dir, self.sensitive_strings_csv, tmp_allowed_binary_csv ) hdf5_searcher.interactive = self.interactive + hdf5_searcher.verify_all_on_behalf_of_user = self.verify_all_on_behalf_of_user hdf5_searcher.date_time_str = self.date_time_str hdf5_searcher.tmp_dir_base = self.tmp_dir_base hdf5_searcher.git_files_only = False @@ -214,12 +226,7 @@ def search_hdf5_file(self, hdf5_file: ff.FileFingerprint): # Ask the user about signing off if self.interactive: - lt.info("Do you want to sign off on this file anyways (y/n)?") - val = input("")[0] - lt.info(f" User responded '{val}'") - if val.lower() == 'y': - matches = [] - else: + if not self.verify_interactively(file_relpath_name_ext): matches.append( ssm.Match(0, 0, 0, "", "", None, "HDF5 file denied by user") ) @@ -252,6 +259,49 @@ def search_hdf5_file(self, hdf5_file: ff.FileFingerprint): return matches + def verify_interactively(self, relative_path_name_ext: str, cv_img: Image.Image = None, cv_title: str = None): + if cv_img is None: + lt.info("") + lt.info("Unknown binary file:") + lt.info(" " + relative_path_name_ext) + lt.info( + "Is this unknown binary file safe to add, and doesn't contain any sensitive information (y/n)?" + ) + if self.verify_all_on_behalf_of_user: + val = 'y' + else: + resp = input("").strip() + val = 'n' if len(resp) == 0 else resp[0] + lt.info(f" User responded '{val}'") + + else: + lt.info("") + lt.info( + "Is this image safe to add, and doesn't contain any sensitive information (y/n)?" + ) + if self.verify_all_on_behalf_of_user: + val = 'y' + else: + cv2.imshow(cv_title, cv_img) + key = cv2.waitKey(0) + cv2.destroyAllWindows() + time.sleep(0.1) # small delay to prevent accidental double-bounces + + # Check for 'y' or 'n' + if key == ord('y') or key == ord('Y'): + val = 'y' + elif key == ord('n') or key == ord('N'): + val = 'n' + else: + val = '?' + if val.lower() in ["y", "n"]: + lt.info(f" User responded '{val}'") + else: + lt.error("Did not respond with either 'y' or 'n'. Assuming 'n'.") + val = 'n' + + return val.lower() == 'y' + def search_binary_file(self, binary_file: ff.FileFingerprint) -> list[ssm.Match]: norm_path = self.norm_path(binary_file.relative_path, binary_file.name_ext) _, _, ext = ft.path_components(norm_path) @@ -273,15 +323,7 @@ def search_binary_file(self, binary_file: ff.FileFingerprint) -> list[ssm.Match] matches += self.search_hdf5_file(binary_file) else: - lt.info("") - lt.info("Unknown binary file:") - lt.info(" " + relative_path_name_ext) - lt.info( - "Is this unknown binary file safe to add, and doesn't contain any sensitive information (y/n)?" - ) - val = input("")[0] - lt.info(f" User responded '{val}'") - if val.lower() != 'y': + if not self.verify_interactively(relative_path_name_ext): matches.append(ssm.Match(0, 0, 0, "", "", None, "Unknown binary file")) return matches @@ -311,13 +353,7 @@ def interactive_image_sign_off( description=f"{file_ff.relative_path}/{file_ff.name_ext}", ) else: - lt.info("Unknown image file failed to open. Do you want to sign off on this file anyways (y/n)?") - val = input("")[0] - lt.info(f" User responded '{val}'") - if val.lower() == 'y': - return True - else: - return False + return self.verify_interactively(file_ff.relative_path) # if img is not None else: return False @@ -338,29 +374,7 @@ def interactive_image_sign_off( rescaled = " (downscaled)" # Show the image and prompt the user - lt.info("") - lt.info( - "Is this image safe to add, and doesn't contain any sensitive information (y/n)?" - ) - cv2.imshow(description + rescaled, np_image) - key = cv2.waitKey(0) - cv2.destroyAllWindows() - time.sleep(0.1) # small delay to prevent accidental double-bounces - - # Check for 'y' or 'n' - if key == ord('y') or key == ord('Y'): - val = 'y' - elif key == ord('n') or key == ord('N'): - val = 'n' - else: - val = '?' - if val.lower() in ["y", "n"]: - lt.info(f" User responded '{val}'") - else: - lt.error("Did not respond with either 'y' or 'n'. Assuming 'n'.") - val = 'n' - - ret = val == 'y' + ret = self.verify_interactively(description, np_image, description+rescaled) return ret def _init_files_lists(self): @@ -395,6 +409,29 @@ def _init_files_lists(self): self.cached_cleared_files.clear() self.new_cached_cleared_files.append(sensitive_strings_cache) + def create_backup_allowed_binaries_csv(self): + path, name, ext = ft.path_components(self.allowed_binary_files_csv) + backup_name_ext = f"{name}_backup_{self.date_time_str}{ext}" + backup_path_name_ext = os.path.join(path, backup_name_ext) + if ft.file_exists(backup_path_name_ext): + ft.delete_file(backup_path_name_ext) + ft.copy_file(self.allowed_binary_files_csv, path, backup_name_ext) + self.has_backed_up_allowed_binaries_csv = True + + def update_allowed_binaries_csv(self): + # Overwrite the allowed list csv file with the updated allowed_binary_files + if not self.has_backed_up_allowed_binaries_csv: + self.create_backup_allowed_binaries_csv() + path, name, ext = ft.path_components(self.allowed_binary_files_csv) + self.allowed_binary_files = sorted(self.allowed_binary_files) + + self.allowed_binary_files[0].to_csv( + "Allowed Binary Files", + path, + name, + rows=self.allowed_binary_files, + ) + def search_files(self): self._init_files_lists() if self.git_files_only: @@ -439,6 +476,13 @@ def search_files(self): else: self._register_file_in_cleared_cache(file_path, file_name_ext) + # Potentially remove unfound binary files + if len(self.unfound_allowed_binary_files) > 0 and self.remove_unfound_binaries: + for file in self.unfound_allowed_binary_files: + self.allowed_binary_files.remove(file) + self.unfound_allowed_binary_files.clear() + self.update_allowed_binaries_csv() + # Print initial information about matching files and problematic binary files if len(matches) > 0: lt.error(f"Found {len(matches)} files containing sensitive strings:") @@ -476,24 +520,9 @@ def search_files(self): self.unknown_binary_files.remove(file_ff) self.allowed_binary_files.append(file_ff) - # First, make a backup copy of the allowed list csv file - if num_signed_binary_files == 0: - path, name, ext = ft.path_components(self.allowed_binary_files_csv) - backup_name_ext = f"{name}_backup_{self.date_time_str}{ext}" - backup_path_name_ext = os.path.join(path, backup_name_ext) - if ft.file_exists(backup_path_name_ext): - ft.delete_file(backup_path_name_ext) - ft.copy_file(self.allowed_binary_files_csv, path, backup_name_ext) - # Overwrite the allowed list csv file with the updated allowed_binary_files - path, name, ext = ft.path_components(self.allowed_binary_files_csv) - self.allowed_binary_files = sorted(self.allowed_binary_files) - file_ff.to_csv( - "Allowed Binary Files", - path, - name, - rows=self.allowed_binary_files, - ) + # and make a backup as necessary. + self.update_allowed_binaries_csv() num_signed_binary_files += 1 @@ -585,8 +614,16 @@ def search_files(self): ) parser.add_argument('--no-interactive', action='store_true', dest="ninteractive", help="Don't interactively ask the user about unknown binary files. Simply fail instead.") + parser.add_argument('--accept-all', action='store_true', dest="acceptall", + help="Don't interactively ask the user about unknown binary files. Simply accept all as verified on the user's behalf. " + + "This can be useful when you're confident that the only changes have been that the binary files have moved but not changed.") + parser.add_argument('--accept-unfound', action='store_true', dest="acceptunfound", + help="Don't fail because of unfound expected binary files. Instead remove the expected files from the list of allowed binaries. " + + "This can be useful when you're confident that the only changes have been that the binary files have moved but not changed.") args = parser.parse_args() not_interactive: bool = args.ninteractive + accept_all: bool = args.acceptall + remove_unfound_binaries: bool = args.acceptunfound ss_log_dir = ft.norm_path( opencsp_settings['sensitive_strings']['sensitive_strings_dir'] @@ -611,6 +648,8 @@ def search_files(self): root_search_dir, sensitive_strings_csv, allowed_binary_files_csv, ss_cache_file ) searcher.interactive = not not_interactive + searcher.verify_all_on_behalf_of_user = accept_all + searcher.remove_unfound_binaries = remove_unfound_binaries searcher.date_time_str = date_time_str num_errors = searcher.search_files() From e578c6bc350fb9fb39aa89f370346bddcfc26f88 Mon Sep 17 00:00:00 2001 From: bbean Date: Tue, 19 Mar 2024 16:36:21 -0600 Subject: [PATCH 11/12] fix plt.figure() not opening sporadically in some of the tests --- opencsp/common/lib/geometry/test/test_LoopXY.py | 3 ++- opencsp/common/lib/geometry/test/test_RegionXY.py | 3 ++- opencsp/common/lib/geometry/test/test_Vxy.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/opencsp/common/lib/geometry/test/test_LoopXY.py b/opencsp/common/lib/geometry/test/test_LoopXY.py index fceec8dc1..cdd4bdba7 100644 --- a/opencsp/common/lib/geometry/test/test_LoopXY.py +++ b/opencsp/common/lib/geometry/test/test_LoopXY.py @@ -6,6 +6,7 @@ from opencsp.common.lib.geometry.LoopXY import LoopXY from opencsp.common.lib.geometry.LineXY import LineXY from opencsp.common.lib.geometry.Vxy import Vxy +import opencsp.common.lib.render.figure_management as fm class TestLoopXY(unittest.TestCase): @@ -124,7 +125,7 @@ def test_draw(self): verts = Vxy(([1, 0, 0, 1], [1, 1, 0, 0])) loop = LoopXY.from_vertices(verts) - fig = plt.figure() + fig = fm._mpl_pyplot_figure() loop.draw(fig.gca()) plt.close(fig) diff --git a/opencsp/common/lib/geometry/test/test_RegionXY.py b/opencsp/common/lib/geometry/test/test_RegionXY.py index a9c41db60..357f0c42f 100644 --- a/opencsp/common/lib/geometry/test/test_RegionXY.py +++ b/opencsp/common/lib/geometry/test/test_RegionXY.py @@ -4,6 +4,7 @@ from opencsp.common.lib.geometry.LoopXY import LoopXY from opencsp.common.lib.geometry.RegionXY import RegionXY from opencsp.common.lib.geometry.Vxy import Vxy +import opencsp.common.lib.render.figure_management as fm class TestRegionXY: @@ -30,6 +31,6 @@ def test_draw(self): loop = LoopXY.from_vertices(verts) region = RegionXY(loop) - fig = plt.figure() + fig = fm._mpl_pyplot_figure() region.draw(fig.gca()) plt.close(fig) diff --git a/opencsp/common/lib/geometry/test/test_Vxy.py b/opencsp/common/lib/geometry/test/test_Vxy.py index b9a7dec8c..28bc66add 100644 --- a/opencsp/common/lib/geometry/test/test_Vxy.py +++ b/opencsp/common/lib/geometry/test/test_Vxy.py @@ -2,6 +2,7 @@ import numpy as np from opencsp.common.lib.geometry.Vxy import Vxy +import opencsp.common.lib.render.figure_management as fm class TestVxy: @@ -262,7 +263,7 @@ def test_cross(self): res = a.cross(b) def test_draw(self): - fig = plt.figure() + fig = fm._mpl_pyplot_figure() ax = plt.gca() self.V1.draw(ax) plt.close(fig) From 8f050a5e410da8a8674a495fbbaf51f54a72621e Mon Sep 17 00:00:00 2001 From: bbean Date: Thu, 21 Mar 2024 10:32:57 -0600 Subject: [PATCH 12/12] reduce requirements.txt pypylon version to what is available on Ubi8, add warning when pypylon version is below the recommended version --- .../lib/camera/ImageAcquisition_DCAM_color.py | 3 +++ .../lib/camera/ImageAcquisition_DCAM_mono.py | 21 +++++++++++++++++++ requirements.txt | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/opencsp/common/lib/camera/ImageAcquisition_DCAM_color.py b/opencsp/common/lib/camera/ImageAcquisition_DCAM_color.py index df50b67f7..44f597699 100644 --- a/opencsp/common/lib/camera/ImageAcquisition_DCAM_color.py +++ b/opencsp/common/lib/camera/ImageAcquisition_DCAM_color.py @@ -3,6 +3,7 @@ from opencsp.common.lib.camera.image_processing import encode_RG_to_RGB from opencsp.common.lib.camera.ImageAcquisitionAbstract import ImageAcquisitionAbstract +from opencsp.common.lib.camera.ImageAcquisition_DCAM_mono import ImageAcquisition as MonoIA class ImageAcquisition(ImageAcquisitionAbstract): @@ -23,6 +24,8 @@ def __init__(self, instance: int = 0, pixel_format: str = 'BayerRG12'): - Other RGB based formats as defined by Basler """ + MonoIA._check_pypylon_version() + # Find all instances of DCAM cameras tlFactory = pylon.TlFactory.GetInstance() devices = tlFactory.EnumerateDevices() diff --git a/opencsp/common/lib/camera/ImageAcquisition_DCAM_mono.py b/opencsp/common/lib/camera/ImageAcquisition_DCAM_mono.py index 304dc2372..b50a957dd 100644 --- a/opencsp/common/lib/camera/ImageAcquisition_DCAM_mono.py +++ b/opencsp/common/lib/camera/ImageAcquisition_DCAM_mono.py @@ -1,10 +1,15 @@ +from importlib.metadata import version import numpy as np + from pypylon import pylon from opencsp.common.lib.camera.ImageAcquisitionAbstract import ImageAcquisitionAbstract +import opencsp.common.lib.tool.log_tools as lt class ImageAcquisition(ImageAcquisitionAbstract): + _has_checked_pypylon_version = False + def __init__(self, instance: int = 0, pixel_format: str = 'Mono8'): """ Class to control a Basler DCAM monochromatic camera. Grabs one frame @@ -22,6 +27,8 @@ def __init__(self, instance: int = 0, pixel_format: str = 'Mono8'): - Others as defined by Basler """ + ImageAcquisition._check_pypylon_version() + # Find all instances of DCAM cameras tlFactory = pylon.TlFactory.GetInstance() devices = tlFactory.EnumerateDevices() @@ -64,6 +71,20 @@ def __init__(self, instance: int = 0, pixel_format: str = 'Mono8'): shutter_min, shutter_max, 2**13 ).astype(int) + @classmethod + def _check_pypylon_version(cls): + if not cls._has_checked_pypylon_version: + pypylon_version = version('pypylon') + suggested_pypylon_version = "3.0" # latest release as of 2024/03/21 + + if pypylon_version < suggested_pypylon_version: + lt.warn("Warning in ImageAcquisition_DCAM_mono.py: " + + f"pypylon version {pypylon_version} is behind the suggested version {suggested_pypylon_version}. " + + "If you have trouble grabbing frames with the basler camera, try upgrading your version of pypylon " + + "with \"python -m pip install --upgrade pypylon\".") + + cls._has_checked_pypylon_version = True + def get_frame(self) -> np.ndarray: # Start frame capture self.cap.StartGrabbingMax(1) diff --git a/requirements.txt b/requirements.txt index 8f78c48d6..22fbee45d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ rawpy >= 0.19.1 openpyxl >= 3.1.2 # these are packages that have errors with both conda and pip on solo #errpkgs -pypylon >= 3.0.1 +pypylon >= 2.3.0 pysolar >= 0.11 tqdm >= 4.66.2