From 6d5750b908f429945600274cd8698711d4661ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pl=C3=ADchal?= Date: Mon, 6 Jan 2025 17:28:34 +0100 Subject: [PATCH 1/2] Preserve order of the `shell` discover tests Tests defined using the `shell` discover method should be executed in the very same order as listed in the discover step config. --- docs/releases.rst | 9 +++++++++ tests/discover/order.sh | 18 ++++++++++++++++++ tests/discover/order/plans.fmf | 12 ++++++++++++ tmt/base.py | 14 +++++++++----- tmt/steps/discover/shell.py | 3 ++- 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/docs/releases.rst b/docs/releases.rst index 3c691f316f..94be241686 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -5,6 +5,15 @@ ====================== +tmt-1.41.0 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Tests defined using the :ref:`/plugins/discover/shell` discover +method are now executed in the exact order as listed in the config +file. This fixes a problem which has been introduced in the recent +``fmf`` update. + + tmt-1.40.0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/discover/order.sh b/tests/discover/order.sh index 9c956559e0..d41ee555a4 100755 --- a/tests/discover/order.sh +++ b/tests/discover/order.sh @@ -177,6 +177,24 @@ EOF EOF run_test + # Tests defined using the 'shell' discover method should be + # executed in the order provided in the config file + + plan="/shell-preserve-order" + + cat > $tmp/EXPECTED-DISCOVERY < $tmp/EXPECTED-EXECUTION <> $TMT_PLAN_DATA/execution_order - name: /order-default test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order + +/shell-preserve-order: + summary: Tests should run in the provided order + discover: + how: shell + tests: + - name: /tests/one + test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order + - name: /tests/two + test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order + - name: /tests/three + test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order diff --git a/tmt/base.py b/tmt/base.py index dcd66954ad..60c0297b07 100644 --- a/tmt/base.py +++ b/tmt/base.py @@ -3033,7 +3033,8 @@ def tests( unique: bool = True, links: Optional[list['LinkNeedle']] = None, excludes: Optional[list[str]] = None, - apply_command_line: bool = True + apply_command_line: bool = True, + sort: bool = True ) -> list[Test]: """ Search available tests """ # Handle defaults, apply possible command line options @@ -3076,8 +3077,8 @@ def name_filter(nodes: Iterable[fmf.Tree]) -> list[fmf.Tree]: if Test._opt('source'): tests = [ - Test(node=test, logger=self._logger.descend()) for test in self.tree.prune( - keys=keys, sources=cmd_line_names)] + Test(node=test, logger=self._logger.descend()) + for test in self.tree.prune(keys=keys, sources=cmd_line_names, sort=sort)] elif not unique and names: # First let's build the list of test objects based on keys & names. @@ -3092,7 +3093,9 @@ def name_filter(nodes: Iterable[fmf.Tree]) -> list[fmf.Tree]: logger=logger.descend( logger_name=test.get('name', None) ) # .apply_verbosity_options(**self._options), - ) for test in name_filter(self.tree.prune(keys=keys, names=[name]))] + ) for test in name_filter( + self.tree.prune(keys=keys, names=[name], sort=sort)) + ] tests.extend(sorted(selected_tests, key=lambda test: test.order)) # Otherwise just perform a regular key/name filtering else: @@ -3103,7 +3106,8 @@ def name_filter(nodes: Iterable[fmf.Tree]) -> list[fmf.Tree]: logger=logger.descend( logger_name=test.get('name', None) ) # .apply_verbosity_options(**self._options), - ) for test in name_filter(self.tree.prune(keys=keys, names=names))] + ) for test in name_filter( + self.tree.prune(keys=keys, names=names, sort=sort))] tests = sorted(selected_tests, key=lambda test: test.order) # Apply filters & conditions diff --git a/tmt/steps/discover/shell.py b/tmt/steps/discover/shell.py index 986f0cd51c..8fd41dd996 100644 --- a/tmt/steps/discover/shell.py +++ b/tmt/steps/discover/shell.py @@ -438,7 +438,8 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None: self._tests = tmt.Tree( logger=self._logger, tree=tests).tests( - conditions=["manual is False"]) + conditions=["manual is False"], + sort=False) # Propagate `where` key and TMT_SOURCE_DIR for test in self._tests: From b4c6b393d5b22f5f297eb1173794cfa27d4563fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pl=C3=ADchal?= Date: Tue, 7 Jan 2025 13:52:26 +0100 Subject: [PATCH 2/2] Require fresh `fmf` with the `sort` support --- .pre-commit-config.yaml | 1 + pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88c5059ad9..4d093982c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -165,6 +165,7 @@ repos: # There is *a lot* of them, and hatch might fetch many of them. - "botocore>=1.25.10" # 1.25.10 is the current one available for RHEL9 - "typing-extensions>=4.9.0; python_version < '3.13'" + - "fmf>=1.6.0" - repo: https://github.com/codespell-project/codespell rev: v2.3.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 53fdfe3f5b..efcdc37c58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ dependencies = [ # F39 / PyPI "click>=8.0.3,!=8.1.4", # 8.1.3 / 8.1.6 TODO type annotations tmt.cli.Context -> click.core.Context click/issues/2558 "docutils>=0.16", # 0.16 is the current one available for RHEL9 - "fmf>=1.4.0", + "fmf>=1.6.0", "jinja2>=2.11.3", # 3.1.2 / 3.1.2 "packaging>=20", # 20 seems to be available with RHEL8 "pint>=0.16.1", # 0.16.1 @@ -90,7 +90,7 @@ docs = [ "readthedocs-sphinx-ext", "docutils>=0.18.1", "Sphinx==7.3.7", - "fmf>=1.4.0", + "fmf>=1.6.0", ] [project.scripts]