From 08d8ad6f7a5fd8f9d0373704cf09883faeb71ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Falc=C3=A3o?= Date: Tue, 26 Dec 2023 02:07:12 +0000 Subject: [PATCH] catches internal errors --- Makefile | 7 ++- docs/source/api-reference.rst | 10 ++-- sure/cli.py | 9 ++- sure/errors.py | 12 +++- sure/{importer.py => loader.py} | 43 ++++++------- sure/meta.py | 4 +- sure/reporter.py | 23 +++++-- sure/reporters/feature.py | 11 +++- sure/runner.py | 4 +- sure/runtime.py | 103 ++++++++++++++++++++------------ tests/unit/__init__.py | 0 tests/unit/test_object_name.py | 26 ++++++++ tests/unit/test_runtime.py | 27 +++++++++ 13 files changed, 198 insertions(+), 81 deletions(-) rename sure/{importer.py => loader.py} (60%) create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/test_object_name.py create mode 100644 tests/unit/test_runtime.py diff --git a/Makefile b/Makefile index db62364..3aa17b7 100644 --- a/Makefile +++ b/Makefile @@ -59,9 +59,10 @@ test tests: clean | $(VENV)/bin/pytest # $(VENV)/bin/nosetests # @$(VENV)/bin/no # run main command-line tool run: | $(MAIN_CLI_PATH) - $(MAIN_CLI_PATH) --with-coverage --cover-branches --cover-module=sure.core --cover-module=sure tests/runner/ - $(MAIN_CLI_PATH) --with-coverage --cover-branches --cover-module=sure.core tests/ - $(MAIN_CLI_PATH) --with-coverage --cover-branches --cover-module=sure.core --immediate + # $(MAIN_CLI_PATH) --with-coverage --cover-branches --cover-module=sure.core tests/ + # $(MAIN_CLI_PATH) --with-coverage --cover-branches --cover-module=sure.core --immediate + # $(MAIN_CLI_PATH) --with-coverage --cover-branches --cover-module=sure.core --cover-module=sure tests/runner/ + $(MAIN_CLI_PATH) --with-coverage --cover-branches --cover-module=sure.runtime tests/unit/ # Pushes release of this package to pypi push-release: dist # pushes distribution tarballs of the current version diff --git a/docs/source/api-reference.rst b/docs/source/api-reference.rst index 5973f6e..8f91bdc 100644 --- a/docs/source/api-reference.rst +++ b/docs/source/api-reference.rst @@ -62,16 +62,16 @@ API Reference .. autoclass:: sure.runner.Runner -``sure.importer`` +``sure.loader`` ----------------- -.. py:module:: sure.importer +.. py:module:: sure.loader -.. autofunction:: sure.importer.resolve_path +.. autofunction:: sure.loader.resolve_path -.. autofunction:: sure.importer.get_root_python_module +.. autofunction:: sure.loader.get_root_python_module -.. autoclass:: sure.importer.importer +.. autoclass:: sure.loader.loader ``sure.reporter`` diff --git a/sure/cli.py b/sure/cli.py index c350fdd..a758c88 100644 --- a/sure/cli.py +++ b/sure/cli.py @@ -29,10 +29,10 @@ import sure.reporters -from sure.importer import resolve_path +from sure.loader import resolve_path from sure.runner import Runner from sure.reporters import gather_reporter_names -from sure.errors import ExitError, ExitFailure +from sure.errors import ExitError, ExitFailure, InternalRuntimeError @click.command(no_args_is_help=True) @@ -65,7 +65,10 @@ def entrypoint(paths, reporter, immediate, log_level, log_file, with_coverage, c cov.start() runner = Runner(resolve_path(os.getcwd()), reporter) - result = runner.run(paths, immediate=immediate) + try: + result = runner.run(paths, immediate=immediate) + except Exception as e: + raise InternalRuntimeError(runner.context, e) if result: if cov: diff --git a/sure/errors.py b/sure/errors.py index ed00d9a..b4af279 100644 --- a/sure/errors.py +++ b/sure/errors.py @@ -14,8 +14,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from __future__ import unicode_literals + import sys +import traceback from functools import reduce @@ -69,3 +70,12 @@ class ExitFailure(ImmediateExit): def __init__(self, context, result): context.reporter.on_failure(result, result.first_nonsuccessful_result) return super().__init__(exit_code('FAILURE')) + + +class InternalRuntimeError(Exception): + def __init__(self, context, exception: Exception): + self.traceback = traceback.format_exc() + self.exception = exception + self.code = exit_code(self.traceback) + super().__init__(self.traceback) + context.reporter.on_internal_runtime_error(context, self) diff --git a/sure/importer.py b/sure/loader.py similarity index 60% rename from sure/importer.py rename to sure/loader.py index 35a9183..eddba0e 100644 --- a/sure/importer.py +++ b/sure/loader.py @@ -14,67 +14,68 @@ def resolve_path(path, relative_to="~") -> Path: return Path(path).absolute().relative_to(Path(relative_to).expanduser()) -def get_root_python_module(path) -> Path: +def get_package(path) -> Path: if not isinstance(path, Path): path = Path(path) if not path.is_dir(): path = path.parent + stack = [] counter = 0 found = None while not found: counter += 1 if not path.parent.joinpath('__init__.py').exists(): found = path + stack.append(found.parent.name) path = path.parent - return found + return found, ".".join(reversed(stack)) -class importer(object): +class loader(object): @classmethod def load_recursive(cls, path, ignore_errors=True, glob_pattern='*.py'): modules = [] path = Path(path) if path.is_file(): - return cls.load_python_file(path) + return cls.load_python_path(path) base_path = Path(path).expanduser().absolute() targets = list(base_path.glob(glob_pattern)) - for file in targets: - modules.extend(cls.load_python_file(file)) + for path in targets: + modules.extend(cls.load_python_path(path)) return modules @classmethod - def load_python_file(cls, file): - if file.is_dir(): - logger.debug(f'ignoring directory {file}') + def load_python_path(cls, path): + if path.is_dir(): + logger.debug(f'ignoring directory {path}') return [] - if file.name.startswith('_') or file.name.endswith('_'): + if path.name.startswith('_') or path.name.endswith('_'): return [] - module, root = cls.dig_to_root(file) - __ROOTS__[str(root)] = root + module, root = cls.traverse_to_package(path) return [module] @classmethod - def dig_to_root(cls, file): - root = get_root_python_module(file) - module_is_artificial = file.parent.joinpath('__init__.py').exists() - module_name = file.parent.name + def traverse_to_package(cls, path): + package, fqdn = get_package(path) + module_is_artificial = path.parent.joinpath('__init__.py').exists() + module_name = path.parent.name if not module_is_artificial: - relative = str(file.relative_to(root.parent)) + relative = str(path.relative_to(root.parent)) module_name = os.path.splitext(relative)[0].replace(os.sep, '.') - spec = importlib.util.spec_from_file_location(module_name, file) + spec = importlib.util.spec_from_file_location(module_name, path) module = importlib.util.module_from_spec(spec) - __MODULES__[module_name] = module - sys.modules[module_name] = module + __MODULES__[fqdn] = module + sys.modules[fqdn] = module try: spec.loader.exec_module(module) except Exception as e: raise e - return module, root.absolute() + return module, package.absolute() diff --git a/sure/meta.py b/sure/meta.py index b965174..d60d4c8 100644 --- a/sure/meta.py +++ b/sure/meta.py @@ -19,7 +19,7 @@ from typing import List from pathlib import Path -from sure.importer import importer +from sure.loader import loader module_root = Path(__file__).parent.absolute() @@ -48,7 +48,7 @@ def internal_module_name(name): def register_class(cls, identifier): cls.kind = identifier - cls.importer = importer + cls.loader = loader if len(cls.__mro__) > 2: register = MODULE_REGISTERS[identifier] return register(cls) diff --git a/sure/reporter.py b/sure/reporter.py index c545e22..9e2b238 100644 --- a/sure/reporter.py +++ b/sure/reporter.py @@ -174,10 +174,25 @@ class SuccessReporter(Reporter): def on_success(self, scenario): sys.stderr.write('Reporter.on_success reported {}'.format(scenario.name)) - class success: - name = 'a simple success' + class FakeScenario: + pass - SuccessReporter('a ').on_success(success) + SuccessReporter('a ').on_success(FakeScenario) + """ + raise NotImplementedError + + def on_internal_runtime_error(self, context, exception: Exception): + """Called when :py:class:`sure.FeatureReporter` + + .. code:: python + + from sure.reporter import Reporter + + class ErrorReporter(Reporter): + def on_internal_runtime_error(self, scenario): + sys.stderr.write('Reporter.on_success reported {}'.format(scenario.name)) + + ErrorReporter('a ').on_internal_runtime_error(context, error) """ raise NotImplementedError @@ -261,7 +276,7 @@ def from_name_and_runner(cls, name, runner): reporter = Reporter.from_name_and_runner('feature', runner) """ - cls.importer.load_recursive( + cls.loader.load_recursive( __path__.joinpath("reporters"), ignore_errors=False, ) diff --git a/sure/reporters/feature.py b/sure/reporters/feature.py index 4ec4b3e..658386f 100644 --- a/sure/reporters/feature.py +++ b/sure/reporters/feature.py @@ -15,6 +15,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + from couleur import Shell from sure.errors import ImmediateFailure @@ -64,7 +65,7 @@ def on_failure(self, test, result): sh.reset("\n") sh.reset(" " * self.indentation) self.indentation += 2 - sh.yellow(f"Failure: {result.succinct_failure}") + sh.bold_yellow(f"Failure:\n{result.succinct_failure}") sh.reset(" " * self.indentation) sh.bold_yellow(f"\n{' ' * self.indentation} Scenario:") sh.bold_yellow(f"\n{' ' * self.indentation} {result.location.description}") @@ -84,10 +85,16 @@ def on_error(self, test, error): sh.red(ballot) sh.reset("\n") sh.reset(" " * self.indentation) - sh.red(" ".join(error.args)) + sh.bold_red(" ".join(error.args)) sh.reset("\n") self.indentation -= 2 + def on_internal_runtime_error(self, context, error): + sh = Shell() + sh.bold_yellow("Internal Runtime Error\n") + sh.bold_red(error.traceback) + raise SystemExit(error.code) + def on_finish(self): failed = len(self.failures) errors = len(self.errors) diff --git a/sure/runner.py b/sure/runner.py index aef47dd..4ac0ae9 100644 --- a/sure/runner.py +++ b/sure/runner.py @@ -41,7 +41,7 @@ stripped, seem_to_indicate_test, ) -from sure.importer import importer +from sure.loader import loader from sure.reporter import Reporter @@ -64,7 +64,7 @@ def get_reporter(self, name): def find_candidates(self, lookup_paths): candidate_modules = [] for path in lookup_paths: - modules = importer.load_recursive(path, glob_pattern="test*.py") + modules = loader.load_recursive(path, glob_pattern="test*.py") candidate_modules.extend(modules) return candidate_modules diff --git a/sure/runtime.py b/sure/runtime.py index 0a2d2cf..384c022 100644 --- a/sure/runtime.py +++ b/sure/runtime.py @@ -35,7 +35,7 @@ ImmediateError, ImmediateFailure, ) -from sure.importer import importer +from sure.loader import loader from sure.reporter import Reporter self = sys.modules[__name__] @@ -89,6 +89,31 @@ def appears_to_be_runnable(name: str) -> bool: ) +class CallGuard(object): + def __init__(self, source, ancestor=None): + self.source = source + self.ancestor = ancestor + self.function = isinstance(source, types.FunctionType) + self.callable = callable(source) + + @property + def name(self): + if self.is_function: + return self.source.__name__ + return self.test.__func__.__name__ + + def __call__(self, *args, **kw): + if isinstance(source, type): + self.source_instance = source() + elif callable(source): + test_methods.insert(0, Container(source.__name__, source, TestLocation( + source, source.__module__ + ), some_object)) + self.source_instance = source.__module__ + else: + raise NotImplementedError(f"PreparedTestSuiteContainer received unexpected type: {source}") + + class TestLocation(object): def __init__(self, test, ancestor=None): self.test = test @@ -96,19 +121,26 @@ def __init__(self, test, ancestor=None): self.filename = self.code.co_filename self.line = self.code.co_firstlineno self.kind = self.test.__class__ - self.name = self.test.__func__.__name__ + self.name = isinstance(self.test, types.FunctionType) and self.test.__name__ or self.test.__func__.__name__ self.ancestor = ancestor self.ancestral_description = "" self.ancestor_repr = "" if ancestor: self.ancestral_description = getattr( ancestor, "description", "" - ) or getattr(ancestor, "__doc__", "") - self.ancestor_repr = ( - f"({self.ancestor.__module__}.{self.ancestor.__name__})" + ) or getattr( + ancestor, "__doc__", "" ) + if isinstance(ancestor, type): + self.ancestor_repr = ( + f"({self.ancestor.__module__}.{self.ancestor.__name__})" + ) + elif isinstance(ancestor, str): + self.ancestor_repr = ancestor + else: + raise NotImplementedError - self.description = self.test.__func__.__doc__ or "" + self.description = getattr(self.test, '__func__', self.test).__doc__ or "" def __repr__(self): return " ".join([self.name, "at", self.ort]) @@ -256,7 +288,18 @@ def __init__( test_methods: List[Callable], nested_suites: List[PreparedTestSuiteContainer], ): - self.source_instance = source() + self.error = None + self.failure = None + if isinstance(source, type): + self.source_instance = source() + elif callable(source): + test_methods.insert(0, Container(source.__name__, source, TestLocation( + source, source.__module__ + ), source.__module__)) + self.source_instance = source.__module__ + else: + raise NotImplementedError(f"PreparedTestSuiteContainer received unexpected type: {source}") + self.log = Logort(self.source_instance) self.context = context self.setup_methods = setup_methods @@ -291,10 +334,10 @@ def from_generic_object(cls, some_object, context: RuntimeContext): if isinstance(some_object, type) and issubclass( some_object, unittest.TestCase ): - # XXX: warn about probability of abuse of TestCase constructor taking wrong arguments + # TODO: warn about probability of abuse of TestCase constructor taking wrong arguments runnable = getattr(some_object(name), name, None) else: - # XXX: support non-unittest.TestCase classes + # TODO: support non-unittest.TestCase classes runnable = getattr(some_object, name, None) # @@ -304,7 +347,7 @@ def from_generic_object(cls, some_object, context: RuntimeContext): ) if seem_to_indicate_setup(name): - # XXX: warn about probability of abuse of TestCase constructor taking non-standard arguments + # TODO: warn about probability of abuse of TestCase constructor taking non-standard arguments setup_methods.append( Container(name, runnable, location, some_object) ) @@ -392,7 +435,6 @@ def run(self, context): yield result, RuntimeRole.Teardown def run_container(self, container, context): - # concentrated area of test execution return self.perform_unit( test=container.unit, context=context, @@ -465,33 +507,21 @@ def run(self, reporter, runtime: RuntimeOptions): class ErrorStack(object): - def __init__(self, exception_info=None): + def __init__(self, location: TestLocation, exc: Exception, exception_info=None): self.exception_info = exception_info or sys.exc_info() self.traceback = self.exception_info[-1] + self.exception = exc + self.location = location - def tb(self): - return self.traceback - - def ff(self): - module_path = str(Path(__file__).parent.absolute()) - tb = self.traceback - cutoff_index = 0 - while True: - yield tb, cutoff_index - code_path = str(Path(tb.tb_frame.f_code.co_filename).absolute()) - if code_path.startswith(module_path): - tb = tb.tb_next - cutoff_index += 1 - else: - yield tb, cutoff_index - break + def location_specific_stack(self): + return [e for e in traceback.format_tb(self.traceback) if self.location.name in e] - def relevant_error_message(self): - stack = list(self.ff()) - return "\n".join(traceback.format_tb(stack[-1][0])) + def location_specific_error(self): + stack = self.location_specific_stack() + return stack and stack[-1] or str(self.exception) - def printable(self): - return "\n".join(self.tb()) + def __str__(self): + return "\n".join(self.location_specific_stack()) class Scenario(object): @@ -538,7 +568,7 @@ def __init__( self.location = location self.context = context self.exc_info = sys.exc_info() - self.stack = ErrorStack(self.exc_info) + self.stack = ErrorStack(location, error, self.exc_info) self.__error__ = None self.__failure__ = None @@ -610,10 +640,7 @@ def succinct_failure(self) -> str: if not self.is_failure: return "" - assertion = self.failure.args[0] - assertion = assertion.replace(self.location.name, "") - assertion = assertion.replace(self.location.ancestor_repr, "") - return assertion.strip() + return self.stack.location_specific_error() class ScenarioResultSet(ScenarioResult): diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_object_name.py b/tests/unit/test_object_name.py new file mode 100644 index 0000000..a212b33 --- /dev/null +++ b/tests/unit/test_object_name.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) <2010-2023> Gabriel Falcão +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"sure.runtime" + +from collections.abc import Awaitable +from sure.runtime import object_name + + +def test_object_name_type(): + "calling ``sure.runtime.object_name(X)`` where X is a ``type``" + assert object_name(Awaitable) == "collections.abc.Awaitable" diff --git a/tests/unit/test_runtime.py b/tests/unit/test_runtime.py new file mode 100644 index 0000000..cd1aadf --- /dev/null +++ b/tests/unit/test_runtime.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) <2010-2023> Gabriel Falcão +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"sure.runtime" + +from collections.abc import Awaitable +from sure.runtime import object_name + + +def test_object_name_type(): + "calling ``sure.runtime.object_name(X)`` where X is a ``type``" + object_name(Awaitable).should_not.equal("collections.abc.Awaitablea") + object_name(Awaitable).should.equal("collections.abc.Awaitable")