From 02c345a7ae50fad8e61a2103a70e3e2fe6f3c1e7 Mon Sep 17 00:00:00 2001 From: Piotr Kosycarz Date: Fri, 17 Jan 2025 11:06:45 +0100 Subject: [PATCH 1/2] scripts: pylib: twister: twisterlib: allow excluding testsuite roots Allow to exclude specific paths from testsuite roots. It takes testsuite roots and filter out one which matches provided pattern, accepting Unix shell-style wildcards. Signed-off-by: Piotr Kosycarz --- scripts/pylib/twister/twisterlib/environment.py | 10 ++++++++++ scripts/pylib/twister/twisterlib/testplan.py | 10 ++++++++++ scripts/tests/twister/test_testplan.py | 3 ++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index e64fb6934460..a2c35dd96755 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -139,6 +139,15 @@ def add_parse_arguments(parser = None) -> argparse.ArgumentParser: "called multiple times. Defaults to the 'samples/' and " "'tests/' directories at the base of the Zephyr tree.") + case_select.add_argument( + "--testsuite-exclude-path", action="append", default=[], type = norm_path, + help="Directory to exclude from searching for test cases. " + "This is a filter pattern for paths provided with the testsuite-root option. " + "Supports Unix shell-style wildcards, e.g. *samples/sub* (fnmatch). " + "The exclude pattern is matched against an absolute path for the test suite. " + "This option can be used multiple times, multiple invocations " + "are treated as a logical 'or' relationship.") + case_select.add_argument( "-f", "--only-failed", @@ -1024,6 +1033,7 @@ def __init__(self, options : argparse.Namespace, default_options=None) -> None: logger.info(f"Using {self.generator}..") self.test_roots = options.testsuite_root + self.test_exclude_paths = options.testsuite_exclude_path if not isinstance(options.board_root, list): self.board_roots = [options.board_root] diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index 5d17f5cc5466..83784e4fc82d 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -7,6 +7,7 @@ # SPDX-License-Identifier: Apache-2.0 import collections import copy +import fnmatch import glob import itertools import json @@ -588,6 +589,15 @@ def add_testsuites(self, testsuite_filter=None): logger.debug("Found possible testsuite in " + dirpath) + skip_path = False + for test_exclude_path in self.env.test_exclude_paths: + if fnmatch.fnmatch(dirpath, test_exclude_path): + skip_path = True + break + if skip_path: + logger.debug(f"Skipping {dirpath} due to excludes") + continue + suite_yaml_path = os.path.join(dirpath, filename) suite_path = os.path.dirname(suite_yaml_path) diff --git a/scripts/tests/twister/test_testplan.py b/scripts/tests/twister/test_testplan.py index 2a006043870e..49514f44512c 100644 --- a/scripts/tests/twister/test_testplan.py +++ b/scripts/tests/twister/test_testplan.py @@ -1451,7 +1451,8 @@ def test_testplan_add_testsuites(tmp_path, testsuite_filter, use_alt_root, detai env = mock.Mock( test_roots=[tmp_test_root_dir], options=mock.Mock(detailed_test_id=detailed_id), - alt_config_root=[tmp_alt_test_root_dir] if use_alt_root else [] + alt_config_root=[tmp_alt_test_root_dir] if use_alt_root else [], + test_exclude_paths=[] ) testplan = TestPlan(env=env) From 4612909e6ce36c5ae945bfc936acf6e52fcac996 Mon Sep 17 00:00:00 2001 From: Piotr Kosycarz Date: Fri, 17 Jan 2025 15:00:00 +0100 Subject: [PATCH 2/2] scripts: ci: test_plan: add option for testsuite excludes Allow to filter out testsuites base on changed files. This is allow for similar way of excluding tests as for tags, however now there will be possible to specify paths for testsuites that should be excluded if certain files were not changed. Signed-off-by: Piotr Kosycarz --- scripts/ci/test_plan.py | 34 +++++++++++++++++++++++++----- scripts/ci/testsuite_excludes.yaml | 17 +++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 scripts/ci/testsuite_excludes.yaml diff --git a/scripts/ci/test_plan.py b/scripts/ci/test_plan.py index 7004ccd8625d..d03211909662 100755 --- a/scripts/ci/test_plan.py +++ b/scripts/ci/test_plan.py @@ -102,7 +102,7 @@ def __repr__(self): class Filters: def __init__(self, modified_files, ignore_path, alt_tags, testsuite_root, - pull_request=False, platforms=[], detailed_test_id=True, quarantine_list=None, tc_roots_th=20): + pull_request=False, platforms=[], detailed_test_id=True, quarantine_list=None, tc_roots_th=20, testsuite_excludes_file=None): self.modified_files = modified_files self.testsuite_root = testsuite_root self.resolved_files = [] @@ -110,6 +110,7 @@ def __init__(self, modified_files, ignore_path, alt_tags, testsuite_root, self.full_twister = False self.all_tests = [] self.tag_options = [] + self.testsuite_excludes_options = [] self.pull_request = pull_request self.platforms = platforms self.detailed_test_id = detailed_test_id @@ -117,10 +118,12 @@ def __init__(self, modified_files, ignore_path, alt_tags, testsuite_root, self.tag_cfg_file = alt_tags self.quarantine_list = quarantine_list self.tc_roots_th = tc_roots_th + self.testsuite_excludes_file = testsuite_excludes_file def process(self): self.find_modules() self.find_tags() + self.find_testsuite_excludes() self.find_tests() if not self.platforms: # disable for now, this is generating lots of churn when changing @@ -326,9 +329,8 @@ def find_tests(self): _options.extend(["-p", platform]) self.get_plan(_options, use_testsuite_root=False) - def find_tags(self): - - with open(self.tag_cfg_file, 'r') as ymlfile: + def _get_tags(self, yml_path): + with open(yml_path, 'r') as ymlfile: tags_config = yaml.safe_load(ymlfile) tags = {} @@ -358,12 +360,29 @@ def find_tags(self): if t.exclude: exclude_tags.add(t.name) + return exclude_tags + + def find_tags(self): + exclude_tags = self._get_tags(self.tag_cfg_file) + for tag in exclude_tags: self.tag_options.extend(["-e", tag ]) if exclude_tags: logging.info(f'Potential tag based filters: {exclude_tags}') + def find_testsuite_excludes(self): + if self.testsuite_excludes_file is None: + return + + exclude_testsuites = self._get_tags(self.testsuite_excludes_file) + + for tag in exclude_testsuites: + self.testsuite_excludes_options.extend(["--testsuite-exclude-path", tag ]) + + if exclude_testsuites: + logging.info(f'Testsuite exclude filters: {exclude_testsuites}') + def find_excludes(self, skip=[]): with open(self.ignore_path, "r") as twister_ignore: ignores = twister_ignore.read().splitlines() @@ -391,9 +410,11 @@ def find_excludes(self, skip=[]): _options.extend(["-p", platform]) _options.extend(self.tag_options) + _options.extend(self.testsuite_excludes_options) self.get_plan(_options) else: _options.extend(self.tag_options) + _options.extend(self.testsuite_excludes_options) self.get_plan(_options, True) else: logging.info(f'No twister needed or partial twister run only...') @@ -443,6 +464,9 @@ def parse_args(): "the file need to correspond to the test scenarios names as in " "corresponding tests .yaml files. These scenarios " "will be skipped with quarantine as the reason.") + parser.add_argument('--testsuite-excludes-file', + default=None, + help="Path to a file describing relations between directories/paths (modified files) and testsuites filters.") # Include paths in names by default. parser.set_defaults(detailed_test_id=True) @@ -472,7 +496,7 @@ def parse_args(): f = Filters(files, args.ignore_path, args.alt_tags, args.testsuite_root, args.pull_request, args.platform, args.detailed_test_id, args.quarantine_list, - args.testcase_roots_threshold) + args.testcase_roots_threshold, args.testsuite_excludes_file) f.process() # remove dupes and filtered cases diff --git a/scripts/ci/testsuite_excludes.yaml b/scripts/ci/testsuite_excludes.yaml new file mode 100644 index 000000000000..56da5d694fed --- /dev/null +++ b/scripts/ci/testsuite_excludes.yaml @@ -0,0 +1,17 @@ +# This file contains information on what files are associated with which +# twister testsuite excludes patterns. +# +# File format is the same as for tags.yaml - please refer to its description. + +# zephyr-keep-sorted-start +"*tests/kernel*": + files: + - kernel/ + - arch/ + - tests/kernel/ + +"*tests/posix*": + files: + - lib/posix/ + - tests/posix/ +# zephyr-keep-sorted-stop