Preserve order of the shell discover tests (#3446)

Tests defined using the `shell` discover method should be executed
in the very same order as listed in the discover step config.

.pre-commit-config.yaml

@@ -165,6 +165,7 @@ repos:
         # There is *a lot* of them, and hatch might fetch many of them.
         - "botocore>=1.25.10"      # 1.25.10 is the current one available for RHEL9
         - "typing-extensions>=4.9.0; python_version < '3.13'"
+        - "fmf>=1.6.0"
   - repo: https://github.com/codespell-project/codespell
     rev: v2.3.0
     hooks:

docs/releases.rst

@@ -5,6 +5,15 @@
 ======================
 
 
+tmt-1.41.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Tests defined using the :ref:`/plugins/discover/shell` discover
+method are now executed in the exact order as listed in the config
+file. This fixes a problem which has been introduced in the recent
+``fmf`` update.
+
+
 tmt-1.40.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

pyproject.toml

@@ -32,7 +32,7 @@ classifiers = [
 dependencies = [              # F39 / PyPI
     "click>=8.0.3,!=8.1.4",   # 8.1.3 / 8.1.6 TODO type annotations tmt.cli.Context -> click.core.Context click/issues/2558
     "docutils>=0.16",         # 0.16 is the current one available for RHEL9
-    "fmf>=1.4.0",
+    "fmf>=1.6.0",
     "jinja2>=2.11.3",         # 3.1.2 / 3.1.2
     "packaging>=20",          # 20 seems to be available with RHEL8
     "pint>=0.16.1",           # 0.16.1
@@ -90,7 +90,7 @@ docs = [
     "readthedocs-sphinx-ext",
     "docutils>=0.18.1",
     "Sphinx==7.3.7",
-    "fmf>=1.4.0",
+    "fmf>=1.6.0",
     ]
 
 [project.scripts]

tests/discover/order.sh

@@ -177,6 +177,24 @@ EOF
 EOF
     run_test
 
+    # Tests defined using the 'shell' discover method should be
+    # executed in the order provided in the config file
+
+    plan="/shell-preserve-order"
+
+    cat > $tmp/EXPECTED-DISCOVERY <<EOF
+/tests/one
+/tests/two
+/tests/three
+EOF
+
+    cat > $tmp/EXPECTED-EXECUTION <<EOF
+/guest/default-0/tests/one-1
+/guest/default-0/tests/two-2
+/guest/default-0/tests/three-3
+EOF
+    run_test
+
     rlPhaseStartCleanup
         rlRun 'rm -rf $tmp' 0 "Remove tmp directory"
     rlPhaseEnd

tests/discover/order/plans.fmf

@@ -90,3 +90,15 @@ finish:
         test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order
       - name: /order-default
         test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order
+
+/shell-preserve-order:
+    summary: Tests should run in the provided order
+    discover:
+        how: shell
+        tests:
+          - name: /tests/one
+            test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order
+          - name: /tests/two
+            test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order
+          - name: /tests/three
+            test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order

tmt/base.py

@@ -3033,7 +3033,8 @@ def tests(
             unique: bool = True,
             links: Optional[list['LinkNeedle']] = None,
             excludes: Optional[list[str]] = None,
-            apply_command_line: bool = True
+            apply_command_line: bool = True,
+            sort: bool = True
             ) -> list[Test]:
         """ Search available tests """
         # Handle defaults, apply possible command line options
@@ -3076,8 +3077,8 @@ def name_filter(nodes: Iterable[fmf.Tree]) -> list[fmf.Tree]:
 
         if Test._opt('source'):
             tests = [
-                Test(node=test, logger=self._logger.descend()) for test in self.tree.prune(
-                    keys=keys, sources=cmd_line_names)]
+                Test(node=test, logger=self._logger.descend())
+                for test in self.tree.prune(keys=keys, sources=cmd_line_names, sort=sort)]
 
         elif not unique and names:
             # First let's build the list of test objects based on keys & names.
@@ -3092,7 +3093,9 @@ def name_filter(nodes: Iterable[fmf.Tree]) -> list[fmf.Tree]:
                         logger=logger.descend(
                             logger_name=test.get('name', None)
                             )  # .apply_verbosity_options(**self._options),
-                        ) for test in name_filter(self.tree.prune(keys=keys, names=[name]))]
+                        ) for test in name_filter(
+                            self.tree.prune(keys=keys, names=[name], sort=sort))
+                    ]
                 tests.extend(sorted(selected_tests, key=lambda test: test.order))
         # Otherwise just perform a regular key/name filtering
         else:
@@ -3103,7 +3106,8 @@ def name_filter(nodes: Iterable[fmf.Tree]) -> list[fmf.Tree]:
                     logger=logger.descend(
                         logger_name=test.get('name', None)
                         )  # .apply_verbosity_options(**self._options),
-                    ) for test in name_filter(self.tree.prune(keys=keys, names=names))]
+                    ) for test in name_filter(
+                        self.tree.prune(keys=keys, names=names, sort=sort))]
             tests = sorted(selected_tests, key=lambda test: test.order)
 
         # Apply filters & conditions

tmt/steps/discover/shell.py

@@ -438,7 +438,8 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None:
         self._tests = tmt.Tree(
             logger=self._logger,
             tree=tests).tests(
-                conditions=["manual is False"])
+                conditions=["manual is False"],
+                sort=False)
 
         # Propagate `where` key and TMT_SOURCE_DIR
         for test in self._tests: