diff --git a/README.md b/README.md index 6649e8c4..e197f355 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,21 @@ python3 test-runner/wasi_test_runner.py -r adapters/wasmtime.py # path to a runtime adapter ``` +You can also optionally override test configs with the `--config-override` +option: + +```bash +python3 test-runner/wasi_test_runner.py \ + -t ./tests/assemblyscript/testsuite/ # path to folders containing .wasm test files \ + ./tests/c/testsuite/ \ + ./tests/rust/testsuite/ \ + --config-override examples/config_override.json \ + -r adapters/wasmtime.py # path to a runtime adapter +``` + +This can be useful for passing additional environment variables to certain +tests without modifying the test config files. + The default executable in the adapter used for test execution can be overridden using `TEST_RUNTIME_EXE` variable. This only works with adapters defined in [adapters/](adapters/), and might not work with 3rd party adapters. diff --git a/examples/config_override.json b/examples/config_override.json new file mode 100644 index 00000000..5e7ec607 --- /dev/null +++ b/examples/config_override.json @@ -0,0 +1,16 @@ +{ + "WASI C tests": { + "stat-dev-ino": { + "dirs": [ + "fs-tests.dir" + ], + "env": { + "TEST_VAR": "test" + }, + "args": [ + "fs-tests.dir", + "test_arg" + ] + } + } +} \ No newline at end of file diff --git a/test-runner/tests/test_test_suite_runner.py b/test-runner/tests/test_test_suite_runner.py index d73438de..20824080 100644 --- a/test-runner/tests/test_test_suite_runner.py +++ b/test-runner/tests/test_test_suite_runner.py @@ -3,6 +3,7 @@ import wasi_test_runner.test_case as tc import wasi_test_runner.test_suite_runner as tsr +from wasi_test_runner.override import StubConfigOverride def get_mock_open() -> Mock: @@ -67,7 +68,9 @@ def test_runner_end_to_end() -> None: filters = [filt] with patch("glob.glob", return_value=test_paths): - suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters) # type: ignore + suite = tsr.run_tests_from_test_suite( + "my-path", runtime, validators, reporters, filters, StubConfigOverride() # type: ignore + ) # Assert manifest was read correctly assert suite.name == "test-suite" @@ -104,6 +107,8 @@ def test_runner_end_to_end() -> None: @patch("os.path.exists", Mock(return_value=False)) def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None: - suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], []) + suite = tsr.run_tests_from_test_suite( + "my-path", Mock(), [], [], [], StubConfigOverride() + ) assert suite.name == "my-path" diff --git a/test-runner/wasi_test_runner/__main__.py b/test-runner/wasi_test_runner/__main__.py index c2b92f45..8dc359c7 100644 --- a/test-runner/wasi_test_runner/__main__.py +++ b/test-runner/wasi_test_runner/__main__.py @@ -11,6 +11,7 @@ from .reporters.console import ConsoleTestReporter from .reporters.json import JSONTestReporter from .validators import exit_code_validator, stdout_validator, Validator +from .override import ConfigOverride, JSONConfigOverride, StubConfigOverride def main() -> int: @@ -46,6 +47,11 @@ def main() -> int: default=False, help="Disables color for console output reporter.", ) + parser.add_argument( + "--config-override", + required=False, + help="Location of JSON file containing overrides for the config used for each test", + ) options = parser.parse_args() @@ -59,12 +65,17 @@ def main() -> int: for filt in options.exclude_filter: filters.append(JSONTestExcludeFilter(filt)) + override: ConfigOverride = StubConfigOverride() + if options.config_override: + override = JSONConfigOverride(options.config_override) + return run_all_tests( RuntimeAdapter(options.runtime_adapter), options.test_suite, validators, reporters, filters, + override, ) diff --git a/test-runner/wasi_test_runner/harness.py b/test-runner/wasi_test_runner/harness.py index 027b2bb5..dd2cac48 100644 --- a/test-runner/wasi_test_runner/harness.py +++ b/test-runner/wasi_test_runner/harness.py @@ -5,6 +5,7 @@ from .test_suite_runner import run_tests_from_test_suite from .runtime_adapter import RuntimeAdapter from .validators import Validator +from .override import ConfigOverride def run_all_tests( @@ -13,12 +14,13 @@ def run_all_tests( validators: List[Validator], reporters: List[TestReporter], filters: List[TestFilter], + override: ConfigOverride, ) -> int: ret = 0 for test_suite_path in test_suite_paths: test_suite = run_tests_from_test_suite( - test_suite_path, runtime, validators, reporters, filters, + test_suite_path, runtime, validators, reporters, filters, override ) for reporter in reporters: reporter.report_test_suite(test_suite) diff --git a/test-runner/wasi_test_runner/override.py b/test-runner/wasi_test_runner/override.py new file mode 100644 index 00000000..fc8c24d6 --- /dev/null +++ b/test-runner/wasi_test_runner/override.py @@ -0,0 +1,45 @@ +import json +from typing import Any, Dict, Optional +from .test_case import ( + Config, +) +from abc import ABC +from abc import abstractmethod + + +class ConfigOverride(ABC): + @abstractmethod + def get_test_override( + self, test_suite_name: str, test_name: str + ) -> Optional[Config]: + pass + + +class JSONConfigOverride(ConfigOverride): + overrides_dict: Dict[str, Dict[str, Dict[str, Any]]] + + def __init__(self, overrides_path: str) -> None: + with open(overrides_path, encoding="utf-8") as file: + self.overrides_dict = json.load(file) + + def get_test_override( + self, test_suite_name: str, test_name: str + ) -> Optional[Config]: + test_suite_overrides = self.overrides_dict.get(test_suite_name) + + if test_suite_overrides is None: + return None + + test_override = test_suite_overrides.get(test_name) + + if test_override is None: + return None + + return Config.from_dict(test_override) + + +class StubConfigOverride(ConfigOverride): + def get_test_override( + self, test_suite_name: str, test_name: str + ) -> Optional[Config]: + return None diff --git a/test-runner/wasi_test_runner/test_case.py b/test-runner/wasi_test_runner/test_case.py index eebb95d3..fc574f6e 100644 --- a/test-runner/wasi_test_runner/test_case.py +++ b/test-runner/wasi_test_runner/test_case.py @@ -37,10 +37,19 @@ class Config(NamedTuple): @classmethod def from_file(cls: Type[T], config_file: str) -> T: - default = cls() - with open(config_file, encoding="utf-8") as file: dict_config = json.load(file) + return cls.from_dict(dict_config) + + @classmethod + def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None: + for field_name in dict_config: + if field_name not in cls._fields: + logging.warning("Unknown field in the config file: %s", field_name) + + @classmethod + def from_dict(cls: Type[T], dict_config: Dict[str, Any]) -> T: + default = cls() cls._validate_dict(dict_config) @@ -53,12 +62,6 @@ def from_file(cls: Type[T], config_file: str) -> T: stdout=dict_config.get("stdout", default.stdout), ) - @classmethod - def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None: - for field_name in dict_config: - if field_name not in cls._fields: - logging.warning("Unknown field in the config file: %s", field_name) - class TestCase(NamedTuple): name: str diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py index c9a26b14..11fd8ff7 100644 --- a/test-runner/wasi_test_runner/test_suite_runner.py +++ b/test-runner/wasi_test_runner/test_suite_runner.py @@ -6,7 +6,7 @@ import time from datetime import datetime -from typing import List, cast +from typing import List, Optional, cast from .filters import TestFilter from .runtime_adapter import RuntimeAdapter @@ -19,6 +19,7 @@ from .reporters import TestReporter from .test_suite import TestSuite from .validators import Validator +from .override import ConfigOverride def run_tests_from_test_suite( @@ -27,6 +28,7 @@ def run_tests_from_test_suite( validators: List[Validator], reporters: List[TestReporter], filters: List[TestFilter], + config_override: ConfigOverride, ) -> TestSuite: test_cases: List[TestCase] = [] test_start = datetime.now() @@ -45,7 +47,12 @@ def run_tests_from_test_suite( test_case = _skip_single_test(runtime, validators, test_path) break else: - test_case = _execute_single_test(runtime, validators, test_path) + test_config_override = config_override.get_test_override( + test_suite_name, test_name + ) + test_case = _execute_single_test( + runtime, validators, test_path, test_config_override + ) test_cases.append(test_case) for reporter in reporters: reporter.report_test(test_case) @@ -73,9 +80,12 @@ def _skip_single_test( def _execute_single_test( - runtime: RuntimeAdapter, validators: List[Validator], test_path: str + runtime: RuntimeAdapter, + validators: List[Validator], + test_path: str, + config_override: Optional[Config], ) -> TestCase: - config = _read_test_config(test_path) + config = config_override or _read_test_config(test_path) test_start = time.time() test_output = runtime.run_test(test_path, config.args, config.env, config.dirs) elapsed = time.time() - test_start