diff --git a/mezcla/debug.py b/mezcla/debug.py index f564c6a7..d5d07610 100755 --- a/mezcla/debug.py +++ b/mezcla/debug.py @@ -721,16 +721,15 @@ def non_debug_stub(*_args, **_kwargs): # Note: no return value assumed by debug.expr return - def get_level(): """Returns tracing level (i.e., 0)""" return trace_level - def get_output_timestamps(): """Non-debug stub""" return False + set_level = non_debug_stub set_output_timestamps = non_debug_stub @@ -807,6 +806,19 @@ def debugging(level=USUAL): ## TODO: use level=WARNING (i.e., 2) return (get_level() >= level) +def active(): + """Whether debugging is active (i.e., trace level 1 or higher) + Note: Use enabled to check whether debugging is supported + """ + return debugging(level=1) + +def enabled(): + """Whether debug code is being executed (i.e., __debug__ nonzero) + Note: Use active to check whether conditional debugging in effect + """ + result = __debug__ + assertion((not result) or active()) + return result def detailed_debugging(): """Whether debugging with trace level DETAILED (4) or higher""" @@ -1176,4 +1188,3 @@ def display_ending_time_etc(): else: ## DEBUG: trace_expr(MOST_VERBOSE, 999) pass - diff --git a/mezcla/glue_helpers.py b/mezcla/glue_helpers.py index 01b89a2d..c1789618 100755 --- a/mezcla/glue_helpers.py +++ b/mezcla/glue_helpers.py @@ -13,7 +13,12 @@ # - Add functions to facilitate functional programming (e.g., to simply debugging traces). # -"""Helpers gluing scripts together""" +"""Helpers gluing scripts together + +Usage example: + + cat {script} | python -c 'from mezcla import glue_helpers, system; print("\\n".join(glue_helpers.elide_values(system.read_lines("{script}"))))' +""" # Standard packages from collections import defaultdict @@ -33,41 +38,60 @@ from mezcla import debug from mezcla import system from mezcla import tpo_common as tpo -from mezcla.tpo_common import debug_format, debug_print, print_stderr, setenv +## OLD: from mezcla.tpo_common import debug_format, debug_print, print_stderr, setenv +from mezcla.tpo_common import debug_format, debug_print + +# Constants +TL = debug.TL +# Environment options +# # note: # - ALLOW_SUBCOMMAND_TRACING should be interepreted in terms of detailed # tracing. Now, basic tracing is still done unless disable_subcommand_tracing() # invoked. (This way, the subscript start/end time is still shown by default) # - SUB_DEBUG_LEVEL added to make sub-script trace level explicit DEFAULT_SUB_DEBUG_LEVEL = int(min(debug.TL.USUAL, debug.get_level())) -SUB_DEBUG_LEVEL = system.getenv_int("SUB_DEBUG_LEVEL", DEFAULT_SUB_DEBUG_LEVEL, - "Tracing level for sub-command scripts invoked") +SUB_DEBUG_LEVEL = system.getenv_int( + "SUB_DEBUG_LEVEL", DEFAULT_SUB_DEBUG_LEVEL, + description="Tracing level for sub-command scripts invoked") default_subtrace_level = SUB_DEBUG_LEVEL -ALLOW_SUBCOMMAND_TRACING = tpo.getenv_boolean("ALLOW_SUBCOMMAND_TRACING", - (SUB_DEBUG_LEVEL > DEFAULT_SUB_DEBUG_LEVEL), - "Whether sub-commands have tracing above TL.USUAL") +ALLOW_SUBCOMMAND_TRACING = system.getenv_boolean( + "ALLOW_SUBCOMMAND_TRACING", + (SUB_DEBUG_LEVEL > DEFAULT_SUB_DEBUG_LEVEL), + description="Whether sub-commands have tracing above TL.USUAL") if ALLOW_SUBCOMMAND_TRACING: # TODO: work out intuitive default if both SUB_DEBUG_LEVEL and ALLOW_SUBCOMMAND_TRACING specified default_subtrace_level = max(debug.get_level(), SUB_DEBUG_LEVEL) -INDENT = " " # default indentation - +INDENT = system.getenv_text( + "INDENT_TEXT", " ", + description="Default indentation") # # note: See main.py for similar support as part of Main scipt class -FILE_BASE = system.getenv_text("FILE_BASE", "_temp", - "Basename for output files including dir") -TEMP_PREFIX = (FILE_BASE + "-") +FILE_BASE = system.getenv_text( + "FILE_BASE", "_temp", + description="Basename for output files including dir") +TEMP_PREFIX = system.getenv_text( + "TEMP_PREFIX", FILE_BASE + "-", + description="Prefix to use for temp files") +TEMP_SUFFIX = system.getenv_text( + "TEMP_SUFFIX", "-", + description="Suffix to use for temp files") +TEMP_SUFFIX = ("-") NTF_ARGS = {'prefix': TEMP_PREFIX, 'delete': not debug.detailed_debugging(), - ## TODO: 'suffix': "-" - } -TEMP_BASE = system.getenv_value("TEMP_BASE", None, - "Override for temporary file basename") + 'suffix': TEMP_SUFFIX} +TEMP_BASE = system.getenv_value( + "TEMP_BASE", None, + description="Override for temporary file basename") TEMP_BASE_DIR_DEFAULT = (TEMP_BASE and (system.is_directory(TEMP_BASE) or TEMP_BASE.endswith("/"))) -USE_TEMP_BASE_DIR = system.getenv_bool("USE_TEMP_BASE_DIR", TEMP_BASE_DIR_DEFAULT, - "Whether TEMP_BASE should be a dir instead of prefix") +USE_TEMP_BASE_DIR = system.getenv_bool( + "USE_TEMP_BASE_DIR", TEMP_BASE_DIR_DEFAULT, + description="Whether TEMP_BASE should be a dir instead of prefix") + +# Globals # note: see init() for initialization TEMP_FILE = None @@ -77,16 +101,16 @@ def get_temp_file(delete=None): """Return name of unique temporary file, optionally with DELETE""" # Note: delete defaults to False if detailed debugging # TODO: allow for overriding other options to NamedTemporaryFile - if ((delete is None) and tpo.detailed_debugging()): + if ((delete is None) and debug.detailed_debugging()): delete = False temp_file_name = (TEMP_FILE or tempfile.NamedTemporaryFile(**NTF_ARGS).name) debug.assertion(not delete, "Support for delete not implemented") debug_format("get_temp_file() => {r}", 5, r=temp_file_name) return temp_file_name # -TEMP_LOG_FILE = tpo.getenv_text("TEMP_LOG_FILE", get_temp_file() + "-log", +TEMP_LOG_FILE = system.getenv_text("TEMP_LOG_FILE", get_temp_file() + "-log", "Log file for stderr such as for issue function") -TEMP_SCRIPT_FILE = tpo.getenv_text("TEMP_SCRIPT_FILE", get_temp_file() + "-script", +TEMP_SCRIPT_FILE = system.getenv_text("TEMP_SCRIPT_FILE", get_temp_file() + "-script", "File for command invocation") def create_temp_file(contents, binary=False): @@ -296,7 +320,7 @@ def indent_lines(text, indentation=None, max_width=512): return result -MAX_ELIDED_TEXT_LEN = tpo.getenv_integer("MAX_ELIDED_TEXT_LEN", 128) +MAX_ELIDED_TEXT_LEN = system.getenv_integer("MAX_ELIDED_TEXT_LEN", 128) # def elide(value, max_len=None): """Returns VALUE converted to text and elided to at most MAX_LEN characters (with '...' used to indicate remainder). @@ -362,16 +386,16 @@ def run(command, trace_level=4, subtrace_level=None, just_issue=None, output=Fal subtrace_level = default_subtrace_level if subtrace_level != trace_level: debug_level_env = os.getenv("DEBUG_LEVEL") - setenv("DEBUG_LEVEL", str(subtrace_level)) + system.setenv("DEBUG_LEVEL", str(subtrace_level)) in_just_issue = just_issue if just_issue is None: just_issue = False save_temp_base = TEMP_BASE if TEMP_BASE: - setenv("TEMP_BASE", TEMP_BASE + "_subprocess_") + system.setenv("TEMP_BASE", TEMP_BASE + "_subprocess_") save_temp_file = TEMP_FILE if TEMP_FILE: - setenv("TEMP_FILE", TEMP_FILE + "_subprocess_") + system.setenv("TEMP_FILE", TEMP_FILE + "_subprocess_") # Expand the command template # TODO: make this optional command_line = command @@ -396,16 +420,17 @@ def run(command, trace_level=4, subtrace_level=None, just_issue=None, output=Fal ## OLD: wait_for_command = (not foreground_wait or not just_issue) wait_for_command = (foreground_wait and not just_issue) debug.trace_expr(5, foreground_wait, just_issue, wait_for_command) + ## TODO3: clarify what output is when stdout redirected (e.g., for issue in support of unittest_wrapper.run_script result = getoutput(command_line) if wait_for_command else str(os.system(command_line)) if output: print(result) # Restore debug level setting in environment if debug_level_env: - setenv("DEBUG_LEVEL", debug_level_env) + system.setenv("DEBUG_LEVEL", debug_level_env) if save_temp_base: - setenv("TEMP_BASE", save_temp_base) + system.setenv("TEMP_BASE", save_temp_base) if save_temp_file: - setenv("TEMP_FILE", save_temp_file) + system.setenv("TEMP_FILE", save_temp_file) debug_print("run(_) => {\n%s\n}" % indent_lines(result), (trace_level + 1)) return result @@ -759,9 +784,9 @@ def getenv_filename(var, default="", description=None): # TODO4: explain motivation debug_format("getenv_filename({v}, {d}, {desc})", 6, v=var, d=default, desc=description) - filename = tpo.getenv_text(var, default, description) + filename = system.getenv_text(var, default, description) if filename and not non_empty_file(filename): - tpo.print_stderr("Error: filename %s empty or missing for environment option %s" % (filename, var)) + system.print_stderr("Error: filename %s empty or missing for environment option %s" % (filename, var)) return filename @@ -823,7 +848,7 @@ def assertion(_condition): def init(): """Work around for Python quirk""" # See https://stackoverflow.com/questions/1590608/how-do-i-forward-declare-a-function-to-avoid-nameerrors-for-functions-defined - debug.trace(5, "gh.init()") + debug.trace(5, "glue_helpers.init()") global TEMP_FILE temp_filename = "temp-file.list" if USE_TEMP_BASE_DIR and TEMP_BASE: @@ -832,9 +857,25 @@ def init(): TEMP_FILE = system.getenv_value("TEMP_FILE", temp_file_default, "Override for temporary filename") +def main(): + """Entry point""" + # Uses dynamic import to avoid circularity + from mezcla.main import Main # pylint: disable=import-outside-toplevel + + # Note: Uses main-based arg parsing for sake of show environment options + # ./glue_helpers.py --help --verbose + debug.trace(TL.USUAL, f"main(): script={system.real_path(__file__)}") + + # Parse command line options, show usage if --help given + main_app = Main(description=__doc__.format(script=basename(__file__))) + debug.assertion(main_app.parsed_args) + return + #------------------------------------------------------------------------ # Warn if invoked standalone # if __name__ == '__main__': - print_stderr("Warning: %s is not intended to be run standalone" % __file__) + debug.trace_current_context(level=TL.QUITE_VERBOSE) + system.print_stderr(f"Warning: {__file__} is not intended to be run standalone\n") + main() diff --git a/mezcla/ipython_utils.py b/mezcla/ipython_utils.py index 5f3b9e0a..d5623ee8 100755 --- a/mezcla/ipython_utils.py +++ b/mezcla/ipython_utils.py @@ -144,8 +144,11 @@ def pr_dir(obj): def set_xterm_title(title=None): - """Set xterm title via set_xterm_title.bash - Note: requires https://github.com/tomasohara/shell-scripts""" + """Set xterm TITLE via set_xterm_title.bash + Note: + - Uses set_xterm_title.bash from https://github.com/tomasohara/shell-scripts. + - The TITLE can use environment variables (e.g., "ipython [$CONDA_PREFIX]"). + """ # Sample result: "ipython: /home/tomohara/mezcla: Py3.10(base)" if title is None: title = "ipython $PWD" diff --git a/mezcla/main.py b/mezcla/main.py index 96961cc9..57150924 100755 --- a/mezcla/main.py +++ b/mezcla/main.py @@ -970,7 +970,7 @@ def clean_up(self): if __name__ == "__main__": system.print_stderr(f"Warning: {__file__} is not intended to be run standalone\n") - # note: Follwing used for argument parsing + # note: Following used for argument parsing main = Main(description=__doc__) main.run() diff --git a/mezcla/system.py b/mezcla/system.py index 63784895..64206012 100755 --- a/mezcla/system.py +++ b/mezcla/system.py @@ -797,12 +797,27 @@ def get_file_modification_time(filename, as_float=False): return mod_time +def split_path(path): + """Split file PATH into directory and filename + Note: wrapper around os.path.split with tracing and sanity checks + """ + # EX: split_path("/etc/passwd") => ["etc", "passwd"] + dir_name, filename = os.path.split(path) + result = dir_name, filename + debug.assertion(dir_name or filename) + if dir_name and debug.active() and file_exists(path): + debug.assertion(directory_exists(dir_name)) + debug.trace(6, f"split_path({path}) => {result}") + return result + def filename_proper(path): - """Return PATH sans directories""" + """Return PATH sans directories + Note: unlike os.path.split, this always returns filename component + """ # EX: filename_proper("/tmp/document.pdf") => "document.pdf") # EX: filename_proper("/tmp") => "tmp") # EX: filename_proper("/") => "/") - (directory, filename) = os.path.split(path) + (directory, filename) = split_path(path) if not filename: filename = directory debug.trace(6, f"filename_proper({path}) => {filename}") diff --git a/mezcla/tests/misc_tests.py b/mezcla/tests/misc_tests.py new file mode 100644 index 00000000..cb1a36a5 --- /dev/null +++ b/mezcla/tests/misc_tests.py @@ -0,0 +1,82 @@ +#! /usr/bin/env python +# +# Miscellaneous tests not tied to particular module. +# +# Note: +# - This is uses to check for enforce some development +# -- A test exists for each module (e.g., tests/test_fubar.py for ./fubar.py). +# -- Python files have execute permissions (e.g., chmod ugo+x). +# + +"""Miscellaneous/non-module tests""" + +# Standard packages +## TODO: from collections import defaultdict + +# Installed packages +import pytest + +# Local packages +from mezcla import debug +from mezcla import glue_helpers as gh +## TODO: from mezcla.my_regex import my_re +from mezcla import system +from mezcla.unittest_wrapper import TestWrapper + +class TestMisc(TestWrapper): + """Class for test case definitions""" + script_module= None + + def get_python_module_files(self, include_tests=False): + """Return list of files for python modules, optionally with INCLUDE_TESTS""" + all_python_modules = gh.run("find . -name '*.py'").splitlines() + ok_python_modules = [] + for module in all_python_modules: + dir_path, _filename = system.split_path(module) + _parent_path, dir_proper = system.split_path(dir_path) + include = True + if (dir_proper == "tests"): + include = include_tests + if include: + ok_python_modules.append(module) + debug.trace_expr(5, ok_python_modules) + return ok_python_modules + + @pytest.mark.xfail + def test_01_check_for_tests(self): + """Make sure test exists for each Python module""" + debug.trace(4, "test_01_check_for_tests()") + for module in self.get_python_module_files(): + dir_name, filename = system.split_path(module) + include = True + # Check for cases to skip (e.g., __init__.py) + if not filename.startswith("__"): + include = False + # Make sure test file exists + if include: + test_path = gh.form_path(dir_name, f"test_{filename}") + self.do_assert(system.file_exists(test_path)) + + @pytest.mark.xfail + def test_02_check_lib_usages(self): + """Make sure modules don't use certain standard library calls directly + Note: The mezcla equivalent should be used for sake of debugging traces""" + debug.trace(4, "test_02_check_lib_usages()") + for module in self.get_python_module_files(): + module_code = system.read_file(module) + bad_usage = False + # Direct use of sys.stdin.read() + # TODO3: use bash AST (e.g., to exclude comments) + if "import sys" in module_code and "sys.stdin.read" in module_code: + bad_usage = (module != "main.py") + debug.trace(4, f"Direct use of sys.stdin.read in {module}") + self.do_assert(not bad_usage) + + @pytest.mark.xfail + def test_03_check_permissions(self): + """Make sure modules don't use certain standard library calls directly + Note: The mezcla equivalent should be used for sake of debugging traces""" + debug.trace(4, "test_03_check_permissions()") + for module in self.get_python_module_files(): + has_execute_perm = gh.run(f'ls -l "{module}" | grep ^...x..x..x') + self.do_assert(has_execute_perm) diff --git a/mezcla/tests/test_debug.py b/mezcla/tests/test_debug.py index 2c84c6b6..e85c8253 100755 --- a/mezcla/tests/test_debug.py +++ b/mezcla/tests/test_debug.py @@ -33,7 +33,8 @@ from mezcla.unittest_wrapper import TestWrapper # Note: Two references are used for the module to be tested: -# THE_MODULE: global module object +# THE_MODULE: global module object +# TestIt.script_module: path to file import mezcla.debug as THE_MODULE # pylint: disable=reimported # Environment options @@ -432,6 +433,7 @@ def test_hidden_simple_trace(self, capsys): class TestDebug2(TestWrapper): """Another Class for test case definitions""" + script_module = TestWrapper.get_testing_module_name(__file__, THE_MODULE) def test_xor3_again(self): """Test xor3 again""" @@ -440,6 +442,14 @@ def test_xor3_again(self): self.do_assert(not debug.xor3(True, True, True)) self.do_assert(not debug.xor3(False, False, False)) + @pytest.mark.xfail + def test_level(self): + """"Make sure set_level honored (provided __debug__)""" + old_level = debug.get_level() + new_level = old_level + 1 + debug.set_level(new_level) + expected_level = (new_level if __debug__ else old_level) + self.do_assert(debug.get_level() == expected_level) #------------------------------------------------------------------------ diff --git a/mezcla/unittest_wrapper.py b/mezcla/unittest_wrapper.py index cf0db53e..83a7ed2c 100755 --- a/mezcla/unittest_wrapper.py +++ b/mezcla/unittest_wrapper.py @@ -157,19 +157,19 @@ class TestWrapper(unittest.TestCase): ## NOTE: leads to pytest warning. See ## https://stackoverflow.com/questions/62460557/cannot-collect-test-class-testmain-because-it-has-a-init-constructor-from ## def __init__(self, *args, **kwargs): - ## debug.trace_fmtd(5, "TestWrapper.__init__({a}): keywords={kw}; self={s}", + ## debug.trace_fmtd(6, "TestWrapper.__init__({a}): keywords={kw}; self={s}", ## a=",".join(args), kw=kwargs, s=self) ## super().__init__(*args, **kwargs) - ## debug.trace_object(5, self, label="TestWrapper instance") + ## debug.trace_object(7, self, label="TestWrapper instance") ## ## __test__ = False # make sure not assumed test @classmethod def setUpClass(cls): """Per-class initialization: make sure script_module set properly""" - debug.trace_fmtd(5, "TestWrapper.setupClass(): cls={c}", c=cls) + debug.trace_fmtd(6, "TestWrapper.setupClass(): cls={c}", c=cls) super().setUpClass() - debug.trace_object(5, cls, "TestWrapper class") + debug.trace_object(7, cls, "TestWrapper class") debug.assertion(cls.script_module != TODO_MODULE) if (cls.script_module is not None): # Try to pull up usage via python -m mezcla.xyz --help @@ -245,7 +245,7 @@ def setUp(self): - Disables tracing scripts invoked via run() unless ALLOW_SUBCOMMAND_TRACING - Initializes temp file name (With override from environment).""" # Note: By default, each test gets its own temp file. - debug.trace(4, "TestWrapper.setUp()") + debug.trace(6, "TestWrapper.setUp()") if not gh.ALLOW_SUBCOMMAND_TRACING: gh.disable_subcommand_tracing() # The temp file is an extension of temp-base file by default. @@ -259,7 +259,7 @@ def setUp(self): gh.delete_existing_file(self.temp_file) TestWrapper.test_num += 1 - debug.trace_object(5, self, "TestWrapper instance") + debug.trace_object(6, self, "TestWrapper instance") return def run_script(self, options=None, data_file=None, log_file=None, trace_level=4, @@ -269,10 +269,11 @@ def run_script(self, options=None, data_file=None, log_file=None, trace_level=4, not specified, they are derived from self.temp_file. The optional POST_OPTIONS go after the data file. Notes: + - OPTIONS uses quotes around shell special characters used (e.g., '<', '>', '|') - issues warning if script invocation leads to error - if USES_STDIN, requires explicit empty string for DATA_FILE to avoid use of - (n.b., as a precaution against hangups)""" debug.trace_fmtd(trace_level + 1, - "TestWrapper.run_script(opts={opts}, data={df}, log={lf}, lvl={lvl}, out={of}, env={env}, stdin={stdin}, post={post}, back={back})", + "TestWrapper.run_script(opts={opts!r}, data={df}, log={lf}, lvl={lvl}, out={of}, env={env}, stdin={stdin}, post={post}, back={back})", opts=options, df=data_file, lf=log_file, lvl=trace_level, of=out_file, env=env_options, stdin=uses_stdin, post=post_options, back=background) if options is None: @@ -311,11 +312,14 @@ def run_script(self, options=None, data_file=None, log_file=None, trace_level=4, # Run the command ## TODO3: allow for stdin_command (e.g., "echo hey" | ...) + ## TODO2: add sanity check for special shell characters + ## shell_tokens = ['<', '>', '|'] + ## debug.assertion(not system.intersection(options.split(), shell_tokens)) gh.issue("{env} python -m {cov_spec} {module} {opts} {path} {post} 1> {out} 2> {log} {amp_spec}", env=env_options, cov_spec=coverage_spec, module=script_module, opts=options, path=data_path, out=out_file, log=log_file, post=post_options, amp_spec=amp_spec) output = system.read_file(out_file) - # note; trailing newline removed as with shell output + # note: trailing newline removed as with shell output if output.endswith("\n"): output = output[:-1] debug.trace_fmtd(trace_level, "output: {{\n{out}\n}}", @@ -412,7 +416,7 @@ def get_stderr(self): def tearDown(self): """Per-test cleanup: deletes temp file unless detailed debugging""" - debug.trace(4, "TestWrapper.tearDown()") + debug.trace(6, "TestWrapper.tearDown()") if not KEEP_TEMP: gh.run("rm -vf {file}*", file=self.temp_file) return @@ -420,7 +424,7 @@ def tearDown(self): @classmethod def tearDownClass(cls): """Per-class cleanup: stub for tracing purposes""" - debug.trace_fmtd(5, "TestWrapper.tearDownClass(); cls={c}", c=cls) + debug.trace_fmtd(6, "TestWrapper.tearDownClass(); cls={c}", c=cls) if not KEEP_TEMP: ## TODO: use shutil if cls.use_temp_base_dir: