Merge pull request #152 from bbean23/151-differentiate-between-exampl…

…e-data-and-large-data-paths

151 differentiate between example data and large data paths

contrib/scripts/__init__.py

@@ -1,15 +0,0 @@
-_ss_settings_key = "sensitive_strings"
-_ss_settings_default: dict[str, None] = {
-    "sensitive_strings_dir": None,
-    "sensitive_strings_file": None,
-    "allowed_binaries_file": None,
-    "cache_file": None,
-}
-"""
-sensitive_strings_dir: Where to save log output to when checking for sensitive strings.
-sensitive_strings_file: Where to find the sensitive_strings.csv files, for use with opencsp_code/contrib/scripts/sensitive_strings.
-allowed_binaries_file: Where to find the sensitive_strings_allowed_binary_files.csv file, for use with opencsp_code/contrib/scripts/sensitive_strings.
-cache_file: Greatly improves the speed of searching for sensitive strings by remembering which files were checked previously, for use with opencsp_code/contrib/scripts/sensitive_strings.
-"""
-
-_settings_list = [[_ss_settings_key, _ss_settings_default]]

contrib/scripts/sensitive_strings.py

@@ -8,8 +8,8 @@
 import numpy as np
 from PIL import Image
 
+from opencsp import opencsp_settings
 import opencsp.common.lib.file.SimpleCsv as sc
-from opencsp.common.lib.opencsp_path import opencsp_settings
 import opencsp.common.lib.opencsp_path.opencsp_root_path as orp
 import opencsp.common.lib.process.subprocess_tools as st
 import opencsp.common.lib.tool.exception_tools as et

opencsp/__init__.py

@@ -12,3 +12,62 @@
 -----
 OpenCSP uses the pytest library.
 """
+
+import configparser
+import os
+
+
+def _opencsp_settings_dirs() -> list[str]:
+    """Returns a list of possible locations for settings files,
+    from lowest to highest importance (higher importance overrides lower importance).
+
+    This function looks for the environmental variable "OPENCSP_SETTINGS_DIRS"
+    and, if "None", returns an empty list (for running unit tests). For any
+    other value of the env var, directories should be delimited with semicolons
+    and are appended to the end of the returned list.
+    """
+    ret: list[str] = []
+
+    # home directories
+    if os.name == "nt":
+        ret.append(os.path.join(os.path.expandvars("%USERPROFILE%"), ".opencsp", "settings"))
+        # TODO add more directories?
+        # ret.append(os.path.join(os.path.expandvars("%LOCALAPPDATA%"), "opencsp", "settings"))
+        # ret.append(os.path.join(os.path.expandvars("%APPDATA%"), "opencsp", "settings"))
+    else:
+        ret.append(os.path.join(os.path.expanduser("~"), ".config", "opencsp", "settings"))
+
+    # environmental directories
+    if "OPENCSP_SETTINGS_DIRS" in os.environ:
+        if os.environ["OPENCSP_SETTINGS_DIRS"] == "None":
+            return []
+        else:
+            additional_dirs = os.environ["OPENCSP_SETTINGS_DIRS"].split(";")
+            for i, dir in enumerate(additional_dirs):
+                additional_dirs[i] = dir.replace("~", os.path.expanduser("~"))
+            ret += additional_dirs
+
+    return ret
+
+
+_settings_files: list[str] = []
+
+# default settings file
+_default_dir = os.path.dirname(__file__)
+_settings_files.append(os.path.join(_default_dir, "default_settings.ini"))
+
+# locate other settings files
+for _dirname in _opencsp_settings_dirs():
+    _settings_file = os.path.join(_dirname, "opencsp_settings.ini")
+    if os.path.exists(_settings_file):
+        _settings_files.append(_settings_file)
+
+# load the settings
+opencsp_settings = configparser.ConfigParser(allow_no_value=True)
+opencsp_settings.read(_settings_files)
+
+for section in opencsp_settings.sections():
+    for key in opencsp_settings[section]:
+        print(f"opencsp_settings[{section}][{key}]={opencsp_settings[section][key]}")
+
+__all__ = ['opencsp_settings']

opencsp/common/lib/cv/SpotAnalysis.py

@@ -7,6 +7,8 @@
 from opencsp.common.lib.cv.spot_analysis.VisualizationCoordinator import VisualizationCoordinator
 import opencsp.common.lib.cv.spot_analysis.image_processor.AbstractSpotAnalysisImageProcessor as asaip
 
+from opencsp import opencsp_settings
+
 # from opencsp.common.lib.cv.spot_analysis.image_processor import * # I suggest importing these dynamically as needed, to reduce startup time
 from opencsp.common.lib.cv.spot_analysis.ImagesIterable import ImagesIterable
 from opencsp.common.lib.cv.spot_analysis.ImagesStream import ImagesStream
@@ -15,7 +17,6 @@
 from opencsp.common.lib.cv.spot_analysis.SpotAnalysisOperablesStream import SpotAnalysisOperablesStream
 from opencsp.common.lib.cv.spot_analysis.SpotAnalysisOperableAttributeParser import SpotAnalysisOperableAttributeParser
 import opencsp.common.lib.render.VideoHandler as vh
-from opencsp.common.lib.opencsp_path import opencsp_settings
 import opencsp.common.lib.opencsp_path.opencsp_root_path as orp
 import opencsp.common.lib.tool.file_tools as ft
 import opencsp.common.lib.tool.image_tools as it

opencsp/common/lib/opencsp_path/__init__.py

@@ -1,165 +0,0 @@
-import copy
-import importlib
-import importlib.util
-import inspect
-import os
-import sys
-
-import opencsp.common.lib.tool.log_tools as lt
-
-_orp_settings_key = "opencsp_root_path"
-_orp_settings_default = {
-    "example_data_dir": None,
-    "scratch_dir": None,
-    "scratch_name": "scratch",
-    "collaborative_dir": None,
-}
-""" example_data_dir: The directory containing the opencsp example data, for examples that have very large data inputs.
-scratch_dir: The directory containing the scratch folder, for use with HPC clusters.
-scratch_name: The name of the scratch directory. Default to "scratch".
-collaborative_dir: A shared directory where experimental data is collected
-"""
-
-_settings_list = [[_orp_settings_key, _orp_settings_default]]
-_opencsp_settings_packages = ["common.lib.tool"]
-_opencsp_code_settings_packages = ["contrib.scripts"]
-
-opencsp_settings = {}
-
-
-def __load_settings_files():
-    """Get the settings for each of the 'settings.json' files found in the
-    _opencsp_settings_dirs(). The settings files should contain a dictionary in
-    JSON form, with group name keys to group dictionaries. For example::
-
-        {
-            'opencsp_root_path': {
-                'scratch_name': 'actual_scratch'
-            }
-        }
-
-    Returns:
-    --------
-    settings_per_file: dict[str,dict[str,any]]
-        The settings names and values, with one dict of values per
-        'settings.json' file. The first key is the file name. The second key is
-        settings group's name ('opencsp_root_path' for this file's settings).
-        The third key, then, is the setting's name.
-    """
-    import os
-    from opencsp.common.lib.opencsp_path.opencsp_root_path import _opencsp_settings_dirs
-    import opencsp.common.lib.tool.file_tools as ft
-
-    ret: dict[str, dict[str, dict[str, any]]] = {}
-
-    # read settings for each file
-    for dir in _opencsp_settings_dirs():
-        settings_file_name_path_ext = os.path.join(dir, 'settings.json')
-
-        # would use file_tools.directory_exists() except that I don't want to depend on any other part of opencsp
-        if os.path.exists(settings_file_name_path_ext) and os.path.isfile(settings_file_name_path_ext):
-            settings_path, settings_name, settings_ext = ft.path_components(settings_file_name_path_ext)
-            settings = ft.read_json("global settings", settings_path, settings_name + settings_ext)
-
-            # verify the types for the loaded settings
-            err_msg_preamble = (
-                f"Error in opencsp_path.__init__(): while parsing settings file {settings_file_name_path_ext}, "
-            )
-            found_err = False
-            if not isinstance(settings, dict):
-                lt.error(
-                    err_msg_preamble
-                    + f"the settings should be a dict but is instead of type {type(group_name)}. Ignoring file."
-                )
-                lt.info(err_msg_preamble + f"'{settings=}'")
-                found_err = True
-
-            for group_name in settings:
-                if not isinstance(group_name, str):
-                    lt.error(
-                        err_msg_preamble
-                        + f"the settings group name should be a string but is instead of type {type(group_name)}. Ignoring file."
-                    )
-                    lt.info(err_msg_preamble + f"'{group_name=}'")
-                    found_err = True
-                    break
-                if not isinstance(settings[group_name], dict):
-                    lt.error(
-                        err_msg_preamble
-                        + f"the settings group should be a dict but is instead of type {type(group_name)}. Ignoring file."
-                    )
-                    lt.info(err_msg_preamble + f"'{settings[group_name]=}'")
-                    found_err = True
-                    break
-
-                for setting_name in settings[group_name]:
-                    if not isinstance(setting_name, str):
-                        lt.error(
-                            err_msg_preamble
-                            + f"the settings name should be a string but is instead of type {type(group_name)}. Ignoring file."
-                        )
-                        lt.info(err_msg_preamble + f"'{setting_name=}'")
-                        found_err = True
-                        break
-                if found_err:
-                    break
-
-            if found_err:
-                continue
-
-            # type checks passed, append to the returned values
-            ret[settings_file_name_path_ext] = settings
-
-    return ret
-
-
-def __populate_settings_list() -> list[tuple[str, dict[str, any]]]:
-    import opencsp
-
-    ret = copy.copy(_settings_list)
-
-    # populate the package settings, which are contained within the "opencsp" package
-    for package_name in _opencsp_settings_packages:
-        package = importlib.import_module("opencsp." + package_name)
-        ret += package._settings_list
-
-    # populate settings from the larger opencsp_code repository
-    opencsp_path = os.path.dirname(inspect.getfile(opencsp))
-    for package_name in _opencsp_code_settings_packages:
-        module_name = "opencsp_code." + package_name
-        package_dir = os.path.abspath(os.path.join(opencsp_path, "..", package_name.replace(".", "/")))
-        if os.path.exists(package_dir):
-            spec = importlib.util.spec_from_file_location(module_name, package_dir + "/__init__.py")
-            module = importlib.util.module_from_spec(spec)
-            sys.modules[module_name] = module
-            spec.loader.exec_module(module)
-            ret += module._settings_list
-
-    return ret
-
-
-def __populate_settings():
-    settings_list = __populate_settings_list()
-
-    # defaults
-    opencsp_settings.clear()
-    for settings_key, settings_defaults in settings_list:
-        if settings_key not in opencsp_settings:
-            opencsp_settings[settings_key] = {}
-        for k in settings_defaults:
-            opencsp_settings[settings_key][k] = settings_defaults[k]
-
-    # override default settings with settings_file-specific settings
-    settings_per_file = __load_settings_files()
-    for settings_file_name_path_ext in settings_per_file:
-        settings = settings_per_file[settings_file_name_path_ext]
-        for settings_key, settings_defaults in settings_list:
-            if settings_key not in settings:
-                continue
-            for k in settings[settings_key]:
-                v = settings[settings_key][k]
-                if v != None:
-                    opencsp_settings[settings_key][k] = v
-
-
-__populate_settings()

opencsp/common/lib/opencsp_path/opencsp_root_path.py

@@ -6,7 +6,7 @@
 import inspect
 import opencsp
 
-from opencsp.common.lib.opencsp_path import opencsp_settings
+from opencsp import opencsp_settings
 import opencsp.common.lib.tool.log_tools as lt
 
 # OpenCSP root directories.
@@ -28,14 +28,22 @@ def opencsp_doc_dir():
     )  # TODO BGB: will this always live next to the 'opencsp' directory?
 
 
-def opencsp_data_example_dir():
+def opencsp_large_data_example_dir():
     """The directory containing the opencsp example data, for examples that have very large data inputs."""
-    example_data_dir: str = opencsp_settings["opencsp_root_path"]["example_data_dir"]
+    example_data_dir: str = opencsp_settings["opencsp_root_path"]["large_data_example_dir"]
     if example_data_dir != None:
         return example_data_dir
     return os.path.join(opencsp_code_dir(), '..', 'opencsp_data_example')
 
 
+def opencsp_example_dir():
+    """The directory containing the opencsp example data, for examples that have small data inputs."""
+    example_data_dir: str = opencsp_settings["opencsp_root_path"]["example_dir"]
+    if example_data_dir != None:
+        return example_data_dir
+    return os.path.join(opencsp_code_dir(), '..', 'example')
+
+
 def opencsp_data_test_dir():
     """This method deprecated. For most tests you can find the data in the neighboring \"data\" directory, inside the \"test\" directory."""
     lt.warn("Deprecation warning (opencsp_data_test_dir): " + opencsp_data_test_dir.__doc__)
@@ -88,40 +96,6 @@ def opencsp_temporary_dir():
     return os.path.join(opencsp_scratch_dir(), "tmp")
 
 
-def _opencsp_settings_dirs():
-    """Returns a list of possible locations for settings files,
-    from highest to lowest importance (higher importance overrides lower importance).
-
-    This function is marked as protected because it is depended on by __init__.py, and
-    so this function can't be dependent on anything in __init__.py.
-
-    This function looks for the environmental variable "OPENCSP_SETTINGS_DIRS"
-    and, if "None", returns an empty list (for running unit tests). For any
-    other value of the env var, directories should be delimited with semicolons
-    and are appended to the end of the returned list.
-    """
-    ret: list[str] = []
-
-    if os.name == "nt":
-        ret.append(os.path.join(os.path.expanduser("~"), ".opencsp", "settings"))
-        # TODO add more directories?
-        # ret.append(os.path.join(os.path.expandvars("%LOCALAPPDATA%"), "opencsp", "settings"))
-        # ret.append(os.path.join(os.path.expandvars("%APPDATA%"), "opencsp", "settings"))
-    else:
-        ret.append(os.path.join(os.path.expanduser("~"), ".config", "opencsp", "settings"))
-
-    if "OPENCSP_SETTINGS_DIRS" in os.environ:
-        if os.environ["OPENCSP_SETTINGS_DIRS"] == "None":
-            return []
-        else:
-            additional_dirs = os.environ["OPENCSP_SETTINGS_DIRS"].split(";")
-            for i, dir in enumerate(additional_dirs):
-                additional_dirs[i] = dir.replace("~", os.path.expanduser("~"))
-            ret += additional_dirs
-
-    return ret
-
-
 def relative_opencsp_data_test_dir():
     """This method deprecated. For most tests you can find the data in the neighboring \"data\" directory, inside the \"test\" directory."""
     lt.warn("Deprecation warning (relative_opencsp_data_test_dir): " + relative_opencsp_data_test_dir.__doc__)

opencsp/common/lib/opencsp_path/test/test_opencsp_root_path.py

@@ -55,7 +55,7 @@ def test_opencsp_doc_dir(self):
 
     def test_opencsp_example_dir(self):
         """Just test that the opencsp_example_dir() method works. TODO actually test the returned value."""
-        self.assertIn("example", orp.opencsp_data_example_dir())
+        self.assertIn("example", orp.opencsp_example_dir())
 
     def test_opencsp_scratch_dir(self):
         """Just test that the opencsp_scratch_dir() method works. TODO actually test the returned value."""
@@ -69,10 +69,6 @@ def test_opencsp_temporary_dir(self):
         """Just test that the opencsp_temporary_dir() method works. TODO actually test the returned value."""
         self.assertTrue(("temp" in orp.opencsp_temporary_dir()) or ("tmp" in orp.opencsp_temporary_dir()))
 
-    def test__opencsp_settings_dirs(self):
-        """Just test that the _opencsp_settings_dirs() method works. TODO actually test the returned value."""
-        orp._opencsp_settings_dirs()
-
     @unittest.skip("Can't get this test to work. Maybe someone smarter than me can make it work? :(")
     @unittest.mock.patch.dict(os.environ, {"OPENCSP_SETTINGS_DIRS": "~/.opencsp/"})
     def test_settings_file(self):

opencsp/common/lib/tool/__init__.py

@@ -1,7 +0,0 @@
-_system_settings_key = "system"
-_system_settings_default = {"is_cluster": True}
-"""
-is_cluster: If this computer is a cluster machine. Should be boolean true or false.
-"""
-
-_settings_list = [[_system_settings_key, _system_settings_default]]

opencsp/common/lib/tool/system_tools.py

@@ -2,6 +2,8 @@
 import re
 import socket
 
+from opencsp import opencsp_settings
+
 
 def is_solo():
     """Determines if this computer is one of the Solo HPC nodes.
@@ -22,9 +24,7 @@ def is_cluster():
     Returns:
         bool: True if running on a HPC cluster node
     """
-    from opencsp.common.lib.opencsp_path import opencsp_settings
-
-    return opencsp_settings['system']['is_cluster'] == True
+    return opencsp_settings['system'].getboolean('is_cluster')
 
 
 __is_production_run = is_cluster() or not __debug__