diff --git a/tests/conftest.py b/tests/conftest.py index e5b22579144..89c214e9bab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,107 @@ from torchvision.tv_tensors import Image, Mask +def pytest_addoption(parser: pytest.Parser): + """Add custom options for perf tests.""" + parser.addoption( + "--model-category", + action="store", + default="all", + choices=("speed", "balance", "accuracy", "default", "other", "all"), + help="Choose speed|balcence|accuracy|default|other|all. Defaults to all.", + ) + parser.addoption( + "--data-group", + action="store", + default="all", + choices=("small", "medium", "large", "all"), + help="Choose small|medium|large|all. Defaults to all.", + ) + parser.addoption( + "--num-repeat", + action="store", + default=0, + help="Overrides default per-data-group number of repeat setting. " + "Random seeds are set to 0 ~ num_repeat-1 for the trials. " + "Defaults to 0 (small=3, medium=3, large=1).", + ) + parser.addoption( + "--num-epoch", + action="store", + default=0, + help="Overrides default per-model number of epoch setting. " + "Defaults to 0 (per-model epoch & early-stopping).", + ) + parser.addoption( + "--eval-upto", + action="store", + default="train", + choices=("train", "export", "optimize"), + help="Choose train|export|optimize. Defaults to train.", + ) + parser.addoption( + "--data-root", + action="store", + default="data", + help="Dataset root directory.", + ) + parser.addoption( + "--output-root", + action="store", + help="Output root directory. Defaults to temp directory.", + ) + parser.addoption( + "--summary-csv", + action="store", + help="Path to output summary cvs file. Defaults to {output-root}/benchmark-summary.csv", + ) + parser.addoption( + "--dry-run", + action="store_true", + default=False, + help="Print OTX commands without execution.", + ) + parser.addoption( + "--deterministic", + action="store_true", + default=False, + help="Turn on deterministic training.", + ) + parser.addoption( + "--user-name", + type=str, + default="anonymous", + help='Sign-off the user name who launched the regression tests this time, e.g., `--user-name "John Doe"`.', + ) + parser.addoption( + "--mlflow-tracking-uri", + type=str, + help="URI for MLFlow Tracking server to store the regression test results.", + ) + parser.addoption( + "--otx-ref", + type=str, + default="__CURRENT_BRANCH_COMMIT__", + help="Target OTX ref (tag / branch name / commit hash) on main repo to test. Defaults to the current branch. " + "`pip install otx[full]@https://github.com/openvinotoolkit/training_extensions.git@{otx_ref}` will be executed before run, " + "and reverted after run. Works only for v2.x assuming CLI compatibility.", + ) + parser.addoption( + "--open-subprocess", + action="store_true", + help="Open subprocess for each CLI test case. " + "This option can be used for easy memory management " + "while running consecutive multiple tests (default: false).", + ) + parser.addoption( + "--task", + action="store", + default="all", + type=str, + help="Task type of OTX to use test.", + ) + + @pytest.fixture(scope="session") def fxt_seg_data_entity() -> tuple[tuple, SegDataEntity, SegBatchDataEntity]: img_size = (32, 32) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 8f387e6ad83..ac61a5d3c7d 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -14,23 +14,6 @@ from tests.test_helpers import find_folder -def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption( - "--open-subprocess", - action="store_true", - help="Open subprocess for each CLI test case. " - "This option can be used for easy memory management " - "while running consecutive multiple tests (default: false).", - ) - parser.addoption( - "--task", - action="store", - default="all", - type=str, - help="Task type of OTX to use test.", - ) - - @pytest.fixture(scope="session") def fxt_ci_data_root() -> Path: data_root = Path(os.environ.get("CI_DATA_ROOT", "/home/validation/data/v2")) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b4dcbd6f1b8..a6223ab4661 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -12,23 +12,6 @@ from otx.core.types.task import OTXTaskType -def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption( - "--open-subprocess", - action="store_true", - help="Open subprocess for each CLI integration test case. " - "This option can be used for easy memory management " - "while running consecutive multiple tests (default: false).", - ) - parser.addoption( - "--task", - action="store", - default="all", - type=str, - help="Task type of OTX to use integration test.", - ) - - @pytest.fixture(scope="module", autouse=True) def fxt_open_subprocess(request: pytest.FixtureRequest) -> bool: """Open subprocess for each CLI integration test case. diff --git a/tests/perf/conftest.py b/tests/perf/conftest.py index 2953a9f7d70..6f031bb5493 100644 --- a/tests/perf/conftest.py +++ b/tests/perf/conftest.py @@ -21,93 +21,6 @@ log = logging.getLogger(__name__) -def pytest_addoption(parser): - """Add custom options for perf tests.""" - parser.addoption( - "--model-category", - action="store", - default="all", - choices=("speed", "balance", "accuracy", "default", "other", "all"), - help="Choose speed|balcence|accuracy|default|other|all. Defaults to all.", - ) - parser.addoption( - "--data-group", - action="store", - default="all", - choices=("small", "medium", "large", "all"), - help="Choose small|medium|large|all. Defaults to all.", - ) - parser.addoption( - "--num-repeat", - action="store", - default=0, - help="Overrides default per-data-group number of repeat setting. " - "Random seeds are set to 0 ~ num_repeat-1 for the trials. " - "Defaults to 0 (small=3, medium=3, large=1).", - ) - parser.addoption( - "--num-epoch", - action="store", - default=0, - help="Overrides default per-model number of epoch setting. " - "Defaults to 0 (per-model epoch & early-stopping).", - ) - parser.addoption( - "--eval-upto", - action="store", - default="train", - choices=("train", "export", "optimize"), - help="Choose train|export|optimize. Defaults to train.", - ) - parser.addoption( - "--data-root", - action="store", - default="data", - help="Dataset root directory.", - ) - parser.addoption( - "--output-root", - action="store", - help="Output root directory. Defaults to temp directory.", - ) - parser.addoption( - "--summary-csv", - action="store", - help="Path to output summary cvs file. Defaults to {output-root}/benchmark-summary.csv", - ) - parser.addoption( - "--dry-run", - action="store_true", - default=False, - help="Print OTX commands without execution.", - ) - parser.addoption( - "--deterministic", - action="store_true", - default=False, - help="Turn on deterministic training.", - ) - parser.addoption( - "--user-name", - type=str, - default="anonymous", - help='Sign-off the user name who launched the regression tests this time, e.g., `--user-name "John Doe"`.', - ) - parser.addoption( - "--mlflow-tracking-uri", - type=str, - help="URI for MLFlow Tracking server to store the regression test results.", - ) - parser.addoption( - "--otx-ref", - type=str, - default="__CURRENT_BRANCH_COMMIT__", - help="Target OTX ref (tag / branch name / commit hash) on main repo to test. Defaults to the current branch. " - "`pip install otx[full]@https://github.com/openvinotoolkit/training_extensions.git@{otx_ref}` will be executed before run, " - "and reverted after run. Works only for v2.x assuming CLI compatibility.", - ) - - @pytest.fixture(scope="session") def fxt_model_category(request: pytest.FixtureRequest) -> str: """Model category to run the benchmark.""" diff --git a/tests/regression/conftest.py b/tests/regression/conftest.py index af5f741ed84..ad716824b61 100644 --- a/tests/regression/conftest.py +++ b/tests/regression/conftest.py @@ -19,33 +19,6 @@ log = logging.getLogger(__name__) -def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption( - "--user-name", - type=str, - required=True, - help="Sign-off the user name who launched the regression tests this time, " 'e.g., `--user-name "John Doe"`.', - ) - parser.addoption( - "--dataset-root-dir", - type=Path, - required=True, - help="Dataset root directory path for the regression tests", - ) - parser.addoption( - "--mlflow-tracking-uri", - type=str, - required=True, - help="URI for MLFlow Tracking server to store the regression test results.", - ) - parser.addoption( - "--num-repeat", - type=int, - default=1, - help="The number of repetitions for each test case with different seed (default=1).", - ) - - @pytest.fixture(scope="module", autouse=True) def fxt_user_name(request: pytest.FixtureRequest) -> str: """User name to sign off the regression test execution. @@ -64,7 +37,7 @@ def fxt_dataset_root_dir(request: pytest.FixtureRequest) -> Path: This should be given by the PyTest CLI option. """ - dataset_root_dir = request.config.getoption("--dataset-root-dir") + dataset_root_dir = request.config.getoption("--data-root") msg = f"dataset_root_dir: {dataset_root_dir}" log.info(msg) return dataset_root_dir