Skip to content

Commit

Permalink
Merge branch 'tom-dev' of https://github.com/tomasohara/mezcla into t…
Browse files Browse the repository at this point in the history
…om-dev
  • Loading branch information
tomasohara committed Jan 13, 2024
2 parents b805270 + c72f0b0 commit ea64a15
Show file tree
Hide file tree
Showing 8 changed files with 217 additions and 51 deletions.
17 changes: 14 additions & 3 deletions mezcla/debug.py
Original file line number Diff line number Diff line change
Expand Up @@ -721,16 +721,15 @@ def non_debug_stub(*_args, **_kwargs):
# Note: no return value assumed by debug.expr
return


def get_level():
"""Returns tracing level (i.e., 0)"""
return trace_level


def get_output_timestamps():
"""Non-debug stub"""
return False

set_level = non_debug_stub

set_output_timestamps = non_debug_stub

Expand Down Expand Up @@ -807,6 +806,19 @@ def debugging(level=USUAL):
## TODO: use level=WARNING (i.e., 2)
return (get_level() >= level)

def active():
"""Whether debugging is active (i.e., trace level 1 or higher)
Note: Use enabled to check whether debugging is supported
"""
return debugging(level=1)

def enabled():
"""Whether debug code is being executed (i.e., __debug__ nonzero)
Note: Use active to check whether conditional debugging in effect
"""
result = __debug__
assertion((not result) or active())
return result

def detailed_debugging():
"""Whether debugging with trace level DETAILED (4) or higher"""
Expand Down Expand Up @@ -1176,4 +1188,3 @@ def display_ending_time_etc():
else:
## DEBUG: trace_expr(MOST_VERBOSE, 999)
pass

105 changes: 73 additions & 32 deletions mezcla/glue_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,12 @@
# - Add functions to facilitate functional programming (e.g., to simply debugging traces).
#

"""Helpers gluing scripts together"""
"""Helpers gluing scripts together
Usage example:
cat {script} | python -c 'from mezcla import glue_helpers, system; print("\\n".join(glue_helpers.elide_values(system.read_lines("{script}"))))'
"""

# Standard packages
from collections import defaultdict
Expand All @@ -33,41 +38,60 @@
from mezcla import debug
from mezcla import system
from mezcla import tpo_common as tpo
from mezcla.tpo_common import debug_format, debug_print, print_stderr, setenv
## OLD: from mezcla.tpo_common import debug_format, debug_print, print_stderr, setenv
from mezcla.tpo_common import debug_format, debug_print

# Constants
TL = debug.TL

# Environment options
#
# note:
# - ALLOW_SUBCOMMAND_TRACING should be interepreted in terms of detailed
# tracing. Now, basic tracing is still done unless disable_subcommand_tracing()
# invoked. (This way, the subscript start/end time is still shown by default)
# - SUB_DEBUG_LEVEL added to make sub-script trace level explicit
DEFAULT_SUB_DEBUG_LEVEL = int(min(debug.TL.USUAL, debug.get_level()))
SUB_DEBUG_LEVEL = system.getenv_int("SUB_DEBUG_LEVEL", DEFAULT_SUB_DEBUG_LEVEL,
"Tracing level for sub-command scripts invoked")
SUB_DEBUG_LEVEL = system.getenv_int(
"SUB_DEBUG_LEVEL", DEFAULT_SUB_DEBUG_LEVEL,
description="Tracing level for sub-command scripts invoked")
default_subtrace_level = SUB_DEBUG_LEVEL
ALLOW_SUBCOMMAND_TRACING = tpo.getenv_boolean("ALLOW_SUBCOMMAND_TRACING",
(SUB_DEBUG_LEVEL > DEFAULT_SUB_DEBUG_LEVEL),
"Whether sub-commands have tracing above TL.USUAL")
ALLOW_SUBCOMMAND_TRACING = system.getenv_boolean(
"ALLOW_SUBCOMMAND_TRACING",
(SUB_DEBUG_LEVEL > DEFAULT_SUB_DEBUG_LEVEL),
description="Whether sub-commands have tracing above TL.USUAL")
if ALLOW_SUBCOMMAND_TRACING:
# TODO: work out intuitive default if both SUB_DEBUG_LEVEL and ALLOW_SUBCOMMAND_TRACING specified
default_subtrace_level = max(debug.get_level(), SUB_DEBUG_LEVEL)

INDENT = " " # default indentation

INDENT = system.getenv_text(
"INDENT_TEXT", " ",
description="Default indentation")
#
# note: See main.py for similar support as part of Main scipt class
FILE_BASE = system.getenv_text("FILE_BASE", "_temp",
"Basename for output files including dir")
TEMP_PREFIX = (FILE_BASE + "-")
FILE_BASE = system.getenv_text(
"FILE_BASE", "_temp",
description="Basename for output files including dir")
TEMP_PREFIX = system.getenv_text(
"TEMP_PREFIX", FILE_BASE + "-",
description="Prefix to use for temp files")
TEMP_SUFFIX = system.getenv_text(
"TEMP_SUFFIX", "-",
description="Suffix to use for temp files")
TEMP_SUFFIX = ("-")
NTF_ARGS = {'prefix': TEMP_PREFIX,
'delete': not debug.detailed_debugging(),
## TODO: 'suffix': "-"
}
TEMP_BASE = system.getenv_value("TEMP_BASE", None,
"Override for temporary file basename")
'suffix': TEMP_SUFFIX}
TEMP_BASE = system.getenv_value(
"TEMP_BASE", None,
description="Override for temporary file basename")
TEMP_BASE_DIR_DEFAULT = (TEMP_BASE and
(system.is_directory(TEMP_BASE) or TEMP_BASE.endswith("/")))
USE_TEMP_BASE_DIR = system.getenv_bool("USE_TEMP_BASE_DIR", TEMP_BASE_DIR_DEFAULT,
"Whether TEMP_BASE should be a dir instead of prefix")
USE_TEMP_BASE_DIR = system.getenv_bool(
"USE_TEMP_BASE_DIR", TEMP_BASE_DIR_DEFAULT,
description="Whether TEMP_BASE should be a dir instead of prefix")

# Globals
# note: see init() for initialization
TEMP_FILE = None

Expand All @@ -77,16 +101,16 @@ def get_temp_file(delete=None):
"""Return name of unique temporary file, optionally with DELETE"""
# Note: delete defaults to False if detailed debugging
# TODO: allow for overriding other options to NamedTemporaryFile
if ((delete is None) and tpo.detailed_debugging()):
if ((delete is None) and debug.detailed_debugging()):
delete = False
temp_file_name = (TEMP_FILE or tempfile.NamedTemporaryFile(**NTF_ARGS).name)
debug.assertion(not delete, "Support for delete not implemented")
debug_format("get_temp_file() => {r}", 5, r=temp_file_name)
return temp_file_name
#
TEMP_LOG_FILE = tpo.getenv_text("TEMP_LOG_FILE", get_temp_file() + "-log",
TEMP_LOG_FILE = system.getenv_text("TEMP_LOG_FILE", get_temp_file() + "-log",
"Log file for stderr such as for issue function")
TEMP_SCRIPT_FILE = tpo.getenv_text("TEMP_SCRIPT_FILE", get_temp_file() + "-script",
TEMP_SCRIPT_FILE = system.getenv_text("TEMP_SCRIPT_FILE", get_temp_file() + "-script",
"File for command invocation")

def create_temp_file(contents, binary=False):
Expand Down Expand Up @@ -296,7 +320,7 @@ def indent_lines(text, indentation=None, max_width=512):
return result


MAX_ELIDED_TEXT_LEN = tpo.getenv_integer("MAX_ELIDED_TEXT_LEN", 128)
MAX_ELIDED_TEXT_LEN = system.getenv_integer("MAX_ELIDED_TEXT_LEN", 128)
#
def elide(value, max_len=None):
"""Returns VALUE converted to text and elided to at most MAX_LEN characters (with '...' used to indicate remainder).
Expand Down Expand Up @@ -362,16 +386,16 @@ def run(command, trace_level=4, subtrace_level=None, just_issue=None, output=Fal
subtrace_level = default_subtrace_level
if subtrace_level != trace_level:
debug_level_env = os.getenv("DEBUG_LEVEL")
setenv("DEBUG_LEVEL", str(subtrace_level))
system.setenv("DEBUG_LEVEL", str(subtrace_level))
in_just_issue = just_issue
if just_issue is None:
just_issue = False
save_temp_base = TEMP_BASE
if TEMP_BASE:
setenv("TEMP_BASE", TEMP_BASE + "_subprocess_")
system.setenv("TEMP_BASE", TEMP_BASE + "_subprocess_")
save_temp_file = TEMP_FILE
if TEMP_FILE:
setenv("TEMP_FILE", TEMP_FILE + "_subprocess_")
system.setenv("TEMP_FILE", TEMP_FILE + "_subprocess_")
# Expand the command template
# TODO: make this optional
command_line = command
Expand All @@ -396,16 +420,17 @@ def run(command, trace_level=4, subtrace_level=None, just_issue=None, output=Fal
## OLD: wait_for_command = (not foreground_wait or not just_issue)
wait_for_command = (foreground_wait and not just_issue)
debug.trace_expr(5, foreground_wait, just_issue, wait_for_command)
## TODO3: clarify what output is when stdout redirected (e.g., for issue in support of unittest_wrapper.run_script
result = getoutput(command_line) if wait_for_command else str(os.system(command_line))
if output:
print(result)
# Restore debug level setting in environment
if debug_level_env:
setenv("DEBUG_LEVEL", debug_level_env)
system.setenv("DEBUG_LEVEL", debug_level_env)
if save_temp_base:
setenv("TEMP_BASE", save_temp_base)
system.setenv("TEMP_BASE", save_temp_base)
if save_temp_file:
setenv("TEMP_FILE", save_temp_file)
system.setenv("TEMP_FILE", save_temp_file)
debug_print("run(_) => {\n%s\n}" % indent_lines(result), (trace_level + 1))
return result

Expand Down Expand Up @@ -759,9 +784,9 @@ def getenv_filename(var, default="", description=None):
# TODO4: explain motivation
debug_format("getenv_filename({v}, {d}, {desc})", 6,
v=var, d=default, desc=description)
filename = tpo.getenv_text(var, default, description)
filename = system.getenv_text(var, default, description)
if filename and not non_empty_file(filename):
tpo.print_stderr("Error: filename %s empty or missing for environment option %s" % (filename, var))
system.print_stderr("Error: filename %s empty or missing for environment option %s" % (filename, var))
return filename


Expand Down Expand Up @@ -823,7 +848,7 @@ def assertion(_condition):
def init():
"""Work around for Python quirk"""
# See https://stackoverflow.com/questions/1590608/how-do-i-forward-declare-a-function-to-avoid-nameerrors-for-functions-defined
debug.trace(5, "gh.init()")
debug.trace(5, "glue_helpers.init()")
global TEMP_FILE
temp_filename = "temp-file.list"
if USE_TEMP_BASE_DIR and TEMP_BASE:
Expand All @@ -832,9 +857,25 @@ def init():
TEMP_FILE = system.getenv_value("TEMP_FILE", temp_file_default,
"Override for temporary filename")

def main():
"""Entry point"""
# Uses dynamic import to avoid circularity
from mezcla.main import Main # pylint: disable=import-outside-toplevel

# Note: Uses main-based arg parsing for sake of show environment options
# ./glue_helpers.py --help --verbose
debug.trace(TL.USUAL, f"main(): script={system.real_path(__file__)}")

# Parse command line options, show usage if --help given
main_app = Main(description=__doc__.format(script=basename(__file__)))
debug.assertion(main_app.parsed_args)
return

#------------------------------------------------------------------------

# Warn if invoked standalone
#
if __name__ == '__main__':
print_stderr("Warning: %s is not intended to be run standalone" % __file__)
debug.trace_current_context(level=TL.QUITE_VERBOSE)
system.print_stderr(f"Warning: {__file__} is not intended to be run standalone\n")
main()
7 changes: 5 additions & 2 deletions mezcla/ipython_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,11 @@ def pr_dir(obj):


def set_xterm_title(title=None):
"""Set xterm title via set_xterm_title.bash
Note: requires https://github.com/tomasohara/shell-scripts"""
"""Set xterm TITLE via set_xterm_title.bash
Note:
- Uses set_xterm_title.bash from https://github.com/tomasohara/shell-scripts.
- The TITLE can use environment variables (e.g., "ipython [$CONDA_PREFIX]").
"""
# Sample result: "ipython: /home/tomohara/mezcla: Py3.10(base)"
if title is None:
title = "ipython $PWD"
Expand Down
2 changes: 1 addition & 1 deletion mezcla/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -970,7 +970,7 @@ def clean_up(self):

if __name__ == "__main__":
system.print_stderr(f"Warning: {__file__} is not intended to be run standalone\n")
# note: Follwing used for argument parsing
# note: Following used for argument parsing
main = Main(description=__doc__)
main.run()

19 changes: 17 additions & 2 deletions mezcla/system.py
Original file line number Diff line number Diff line change
Expand Up @@ -797,12 +797,27 @@ def get_file_modification_time(filename, as_float=False):
return mod_time


def split_path(path):
"""Split file PATH into directory and filename
Note: wrapper around os.path.split with tracing and sanity checks
"""
# EX: split_path("/etc/passwd") => ["etc", "passwd"]
dir_name, filename = os.path.split(path)
result = dir_name, filename
debug.assertion(dir_name or filename)
if dir_name and debug.active() and file_exists(path):
debug.assertion(directory_exists(dir_name))
debug.trace(6, f"split_path({path}) => {result}")
return result

def filename_proper(path):
"""Return PATH sans directories"""
"""Return PATH sans directories
Note: unlike os.path.split, this always returns filename component
"""
# EX: filename_proper("/tmp/document.pdf") => "document.pdf")
# EX: filename_proper("/tmp") => "tmp")
# EX: filename_proper("/") => "/")
(directory, filename) = os.path.split(path)
(directory, filename) = split_path(path)
if not filename:
filename = directory
debug.trace(6, f"filename_proper({path}) => {filename}")
Expand Down
82 changes: 82 additions & 0 deletions mezcla/tests/misc_tests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
#! /usr/bin/env python
#
# Miscellaneous tests not tied to particular module.
#
# Note:
# - This is uses to check for enforce some development
# -- A test exists for each module (e.g., tests/test_fubar.py for ./fubar.py).
# -- Python files have execute permissions (e.g., chmod ugo+x).
#

"""Miscellaneous/non-module tests"""

# Standard packages
## TODO: from collections import defaultdict

# Installed packages
import pytest

# Local packages
from mezcla import debug
from mezcla import glue_helpers as gh
## TODO: from mezcla.my_regex import my_re
from mezcla import system
from mezcla.unittest_wrapper import TestWrapper

class TestMisc(TestWrapper):
"""Class for test case definitions"""
script_module= None

def get_python_module_files(self, include_tests=False):
"""Return list of files for python modules, optionally with INCLUDE_TESTS"""
all_python_modules = gh.run("find . -name '*.py'").splitlines()
ok_python_modules = []
for module in all_python_modules:
dir_path, _filename = system.split_path(module)
_parent_path, dir_proper = system.split_path(dir_path)
include = True
if (dir_proper == "tests"):
include = include_tests
if include:
ok_python_modules.append(module)
debug.trace_expr(5, ok_python_modules)
return ok_python_modules

@pytest.mark.xfail
def test_01_check_for_tests(self):
"""Make sure test exists for each Python module"""
debug.trace(4, "test_01_check_for_tests()")
for module in self.get_python_module_files():
dir_name, filename = system.split_path(module)
include = True
# Check for cases to skip (e.g., __init__.py)
if not filename.startswith("__"):
include = False
# Make sure test file exists
if include:
test_path = gh.form_path(dir_name, f"test_{filename}")
self.do_assert(system.file_exists(test_path))

@pytest.mark.xfail
def test_02_check_lib_usages(self):
"""Make sure modules don't use certain standard library calls directly
Note: The mezcla equivalent should be used for sake of debugging traces"""
debug.trace(4, "test_02_check_lib_usages()")
for module in self.get_python_module_files():
module_code = system.read_file(module)
bad_usage = False
# Direct use of sys.stdin.read()
# TODO3: use bash AST (e.g., to exclude comments)
if "import sys" in module_code and "sys.stdin.read" in module_code:
bad_usage = (module != "main.py")
debug.trace(4, f"Direct use of sys.stdin.read in {module}")
self.do_assert(not bad_usage)

@pytest.mark.xfail
def test_03_check_permissions(self):
"""Make sure modules don't use certain standard library calls directly
Note: The mezcla equivalent should be used for sake of debugging traces"""
debug.trace(4, "test_03_check_permissions()")
for module in self.get_python_module_files():
has_execute_perm = gh.run(f'ls -l "{module}" | grep ^...x..x..x')
self.do_assert(has_execute_perm)
Loading

0 comments on commit ea64a15

Please sign in to comment.