define parameter combinations

tests/conftest.py

@@ -36,10 +36,10 @@ def pytest_addoption(parser: argparsing.Parser) -> None:
 
 def pytest_configure(config: Config) -> None:
     config.addinivalue_line("markers", "slow: mark test as slow to run")
-    config.addinivalue_line(
-        "markers",
-        "subprocess_test: mark a test to run only as a subprocess within another test",
-    )
+    #config.addinivalue_line(
+    #    "markers",
+    #    "subprocess_test: mark a test to run only as a subprocess within another test",
+    #)
     config.addinivalue_line(
         "markers",
         "release: mark a test to run only for release",

tests/release/conftest.py

@@ -53,7 +53,6 @@
 FULL_USA_FILEPATH = "/mnt/team/simulation_science/pub/models/vivarium_census_prl_synth_pop/results/release_02_yellow/full_data/united_states_of_america/2023_08_21_16_35_27/final_results/2023_08_31_15_58_01/pseudopeople_simulated_population_usa_2_0_0"
 RI_FILEPATH = "/mnt/team/simulation_science/pub/models/vivarium_census_prl_synth_pop/results/release_02_yellow/full_data/united_states_of_america/2023_08_21_16_35_27/final_results/2023_08_31_15_58_01/states/pseudopeople_simulated_population_rhode_island_2_0_0"
 SOURCE_MAPPER = {"usa": FULL_USA_FILEPATH, "ri": RI_FILEPATH, "sample": None}
-DEFAULT_CONFIG = None
 
 
 def pytest_addoption(parser: pytest.Parser) -> None:
@@ -87,12 +86,6 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         default=CLI_DEFAULT_ENGINE,
         help="The engine used to generate data. Options are 'pandas' and 'dask'.",
     )
-    parser.addoption(
-        "--config",
-        action="store",
-        default=DEFAULT_CONFIG,
-        help="The noise config to use when generating data.",
-    )
 
 
 ############
@@ -114,7 +107,7 @@ def release_output_dir() -> Path:
 def dataset_params(
     request: pytest.FixtureRequest,
 ) -> tuple[str | int | Callable[..., pd.DataFrame] | None, ...]:
-    dataset_name = request.config.getoption("--dataset")
+    dataset_name = request.config.getoption("--dataset", default=CLI_DEFAULT_DATASET)
     try:
         dataset_func = DATASET_GENERATION_FUNCS[dataset_name]
     except KeyError:
@@ -140,6 +133,10 @@ def dataset_params(
 
 @pytest.fixture(scope="session")
 def data(release_output_dir: Path, request: pytest.FixtureRequest, config: dict[str, Any]) -> pd.DataFrame:
+    marker = request.config.getoption('-m')
+    if marker != 'release':
+        return 0
+
     _, dataset_func, source, year, state, engine = request.getfixturevalue("dataset_params")
 
     if source is None:
@@ -162,6 +159,10 @@ def unnoised_dataset(
     request: pytest.FixtureRequest,
     config: dict[str, Any],
 ) -> pd.DataFrame:
+    marker = request.config.getoption('-m')
+    if marker != 'release':
+        return 0
+
     dataset_arg, dataset_func, source, year, state, engine = dataset_params
     dataset_name = DATASET_ARG_TO_FULL_NAME_MAPPER[dataset_arg]
 
@@ -185,7 +186,7 @@ def unnoised_dataset(
 
 @pytest.fixture(scope="session")
 def dataset_name(request: pytest.FixtureRequest) -> str:
-    dataset_arg = request.config.getoption("--dataset")
+    dataset_arg = request.config.getoption("--dataset", default=CLI_DEFAULT_DATASET)
     return DATASET_ARG_TO_FULL_NAME_MAPPER[dataset_arg]
 
 

tests/release/test_fixture_args.py

@@ -71,7 +71,6 @@ def check_subprocess_environment() -> None:
 }
 
 
-@pytest.mark.subprocess_test
 @pytest.mark.usefixtures("check_subprocess_environment")
 def test_parsing_fixture_params(request: pytest.FixtureRequest) -> None:
     output = request.getfixturevalue("dataset_params")

tests/release/test_release.py

@@ -96,6 +96,7 @@ def test_row_noising_omit_row_or_do_not_respond(
     run_omit_row_or_do_not_respond_tests(dataset_name, config, original_data, noised_data)
 
 
+@pytest.mark.release
 def test_unnoised_id_cols(dataset_name: str, request: FixtureRequest) -> None:
     """Tests that all datasets retain unnoised simulant_id and household_id
     (except for SSA which does not include household_id)

tests/release/test_runner.py

@@ -0,0 +1,23 @@
+import os
+import subprocess
+import pytest
+from pathlib import Path
+
+@pytest.mark.parametrize(
+    "pytest_args",
+    [
+        ([]),
+        (["--dataset", "acs"]),
+        (["--dataset", "cps"]),
+        #(["--dataset", "acs", "--population", "USA"]),
+        #(["--dataset", "acs", "--population", "USA", "--state", "RI"]),
+        (["--dataset", "cps", "--year", "2015"]),
+        #(["--dataset", "wic", "--population", "USA", "--state", "RI", "--year", "2015"]),
+    ],
+)
+def test_integration(pytest_args: list[str]) -> None:
+    os.chdir(Path(__file__).parent)  # need this to access options from conftest.py
+    base_cmd = ["pytest", "-m", "release", "test_release.py"]
+    cmd = base_cmd + pytest_args
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    assert result.returncode == 0