diff --git a/opencsp/common/lib/process/MultiprocessNonDaemonic.py b/opencsp/common/lib/process/MultiprocessNonDaemonic.py index e1d6af2e9..e9ab6a961 100644 --- a/opencsp/common/lib/process/MultiprocessNonDaemonic.py +++ b/opencsp/common/lib/process/MultiprocessNonDaemonic.py @@ -4,26 +4,29 @@ class MultiprocessNonDaemonic: + """ + A class for managing a pool of non-daemonic processes for parallel execution. + + This class is similar to `multiprocessing.Pool`, but it allows for the creation of non-daemonic + processes, which can spawn child processes. This is useful in scenarios where grandchild processes + need to be created from the child processes. + """ + + # "ChatGPT 4o" assisted with generating this docstring. def __init__(self, num_processes: int): - """This class is like multiprocessing.Pool, but the processes it uses aren't daemonic. - - Some properties of daemonic processes include: - - When a process exits, it attempts to terminate all of its daemonic child processes. - - Note that a daemonic process is not allowed to create child processes. Otherwise a - daemonic process would leave its children orphaned if it gets terminated when its - parent process exits. Additionally, these are not Unix daemons or services, they - are normal processes that will be terminated (and not joined) if non-daemonic - processes have exited. - - The second point (not being able to spawn child processes) is why this class exists. - Sometimes you want to be able to spawn grandchild processes from the child processes - started with Multiprocessing.starmap(). One example use case is to spawn a grandchild - process that handles rendering for the child process. - - Args: + """Initializes the MultiprocessNonDaemonic instance. + + Parameters + ---------- + num_processes : int + The number of processes in this pool. + + Notes ----- - num_processes (int): The number of processes in this pool. + Daemonic processes have certain limitations, such as not being able to create child processes. + This class allows for the creation of non-daemonic processes to facilitate such use cases. """ + # "ChatGPT 4o" assisted with generating this docstring. self.procs: list[multiprocessing.Process] = [] self.num_processes = num_processes self.queue = multiprocessing.Queue() @@ -52,8 +55,30 @@ def _do_work(func, queue, i, vargs): queue.put([i, ret]) def starmap(self, func: Callable, args: Iterable[Iterable]): - results = [] - + """ + Distributes the execution of a function across multiple processes. + + This method takes a function and a sequence of argument tuples, and executes the function + in parallel using the specified number of processes. + + Parameters + ---------- + func : Callable + The function to execute in parallel. + args : Iterable[Iterable] + An iterable of argument tuples to pass to the function. + + Returns + ------- + list + A list of results returned by the function, in the order of the input arguments. + + Raises + ------ + AssertionError + If the number of processes exceeds the specified limit. + """ + # "ChatGPT 4o" assisted with generating this docstring. for proc_idx, proc_args in enumerate(args): # wait for a process slot to become available while len(self.procs) >= self.num_processes: diff --git a/opencsp/common/lib/process/ServerSynchronizer.py b/opencsp/common/lib/process/ServerSynchronizer.py index ffa9493d9..b6275a25c 100644 --- a/opencsp/common/lib/process/ServerSynchronizer.py +++ b/opencsp/common/lib/process/ServerSynchronizer.py @@ -10,6 +10,11 @@ class ServerSynchronizer: + """ + Helper class to force all servers to wait at specified synchronization points. + This is particularly useful for scatter-gather type workflows. + """ + path = os.path.join(orp.opencsp_temporary_dir(), "synchronize_servers_by_file") def __init__( diff --git a/opencsp/common/lib/process/subprocess_tools.py b/opencsp/common/lib/process/subprocess_tools.py index e4b83325c..4a2ba05eb 100644 --- a/opencsp/common/lib/process/subprocess_tools.py +++ b/opencsp/common/lib/process/subprocess_tools.py @@ -45,6 +45,28 @@ def get_executable_path(executable_name: str, dont_match: str = None) -> str: def filter_lines(lines: list[pol.ProcessOutputLine], keep_stdout=True, keep_stderr=True): + """ + Filters a list of process output lines based on specified criteria. + + This function allows you to filter out standard output (stdout) or standard error (stderr) lines + from a list of process output lines. You can choose to keep only the stdout lines, only the stderr + lines, or both. + + Parameters + ---------- + lines : list[pol.ProcessOutputLine] + A list of process output lines to filter. + keep_stdout : bool, optional + If True, keeps the stdout lines. If False, removes them. Defaults to True. + keep_stderr : bool, optional + If True, keeps the stderr lines. If False, removes them. Defaults to True. + + Returns + ------- + list[pol.ProcessOutputLine] + A list of filtered process output lines based on the specified criteria. + """ + # "ChatGPT 4o" assisted with generating this docstring. ret: list[pol.ProcessOutputLine] = list(lines) if not keep_stdout: @@ -58,6 +80,22 @@ def filter_lines(lines: list[pol.ProcessOutputLine], keep_stdout=True, keep_stde def print_lines(lines: list[pol.ProcessOutputLine]): + """ + Prints the process output lines to the console. + + This function iterates through a list of process output lines and prints each line to the console. + If the line is an error line, it uses the error logging function; otherwise, it uses the info logging function. + + Parameters + ---------- + lines : list[pol.ProcessOutputLine] + A list of process output lines to print. + + Returns + ------- + None + """ + # "ChatGPT 4o" assisted with generating this docstring. for line in lines: if line.is_err: lt.error(line.val) diff --git a/opencsp/test/test_DocStringsExist.py b/opencsp/test/test_DocStringsExist.py index b8035f0cb..8b461aa94 100644 --- a/opencsp/test/test_DocStringsExist.py +++ b/opencsp/test/test_DocStringsExist.py @@ -26,6 +26,8 @@ import opencsp.common.lib.geometry.TranslationXYZ as TranslationXYZ import opencsp.common.lib.geometry.matrix_geometry_3d as matrix_geometry_3d import opencsp.common.lib.opencsp_path.optical_analysis_data_path as optical_analysis_data_path +import opencsp.common.lib.process.ServerSynchronizer as ServerSynchronizer +import opencsp.common.lib.process.parallel_video_tools as parallel_video_tools def test_docstrings_exist_for_methods(): @@ -221,6 +223,15 @@ def test_docstrings_exist_for_methods(): opencsp.common.lib.photogrammetry.photogrammetry, ] + process_class_list = [ + opencsp.common.lib.process.MultiprocessNonDaemonic, + opencsp.common.lib.process.ParallelPartitioner, + opencsp.common.lib.process.ServerSynchronizer, + opencsp.common.lib.process.parallel_file_tools, + opencsp.common.lib.process.parallel_video_tools, + opencsp.common.lib.process.subprocess_tools, + ] + common_class_list = ( camera_class_list + csp_class_list @@ -231,6 +242,7 @@ def test_docstrings_exist_for_methods(): + geometry_class_list + opencsp_path_class_list + photogrammetry_class_list + + process_class_list ) class_list = app_class_list + common_class_list