test: adds timeout and skips some breaking integration tests (#1890)

* test: adds pytest-timeout to prevent some failing tests from running forever

* fix(test): skip plugin install integration tests because they reinstall ape

* test: make skip reason string clearer

* test: adds --includepip pytest option to skip tests marked as pip

* test: add `@run_once` decorator to pip marked tests

pyproject.toml

@@ -29,7 +29,9 @@ norecursedirs = "projects"
 addopts = "-p no:ape_test"  # NOTE: Prevents the ape plugin from activating on our tests
 python_files = "test_*.py"
 testpaths = "tests"
-markers = "fuzzing: Run Hypothesis fuzz test suite"
+markers = """fuzzing: Run Hypothesis fuzz test suite
+pip: tests that rely on pip install operations"""
+timeout = 300
 
 [tool.isort]
 line_length = 100

setup.py

@@ -14,6 +14,7 @@
         "pytest-xdist>=3.5.0,<4",  # Multi-process runner
         "pytest-cov>=4.0.0,<5",  # Coverage analyzer plugin
         "pytest-mock",  # For creating mocks
+        "pytest-timeout~=2.2.0",
         "hypothesis>=6.2.0,<7.0",  # Strategy-based fuzzer
         "hypothesis-jsonschema==0.19.0",  # JSON Schema fuzzer extension
     ],

tests/conftest.py

@@ -35,6 +35,17 @@
 explorer_test = pytest.mark.xdist_group(name="explorer-tests")
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--includepip", action="store_true", help="run tests that depend on pip install operations"
+    )
+
+
+def pytest_runtest_setup(item):
+    if "pip" in item.keywords and not item.config.getoption("--includepip"):
+        pytest.skip("need --includepip option to run this test")
+
+
 @pytest.fixture(autouse=True)
 def setenviron(monkeypatch):
     """

tests/integration/cli/test_plugins.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from tests.integration.cli.utils import github_xfail
+from tests.integration.cli.utils import github_xfail, run_once
 
 TEST_PLUGIN_NAME = "tokens"
 TEST_PLUGIN_NAME_2 = "optimism"
@@ -129,25 +129,29 @@ def test_list_does_not_repeat(ape_plugins_runner, installed_plugin):
     assert "ethereum" not in result.available_plugins
 
 
-@github_xfail()
+@pytest.mark.pip
+@run_once
 def test_upgrade(ape_plugins_runner, installed_plugin):
     result = ape_plugins_runner.invoke(["install", TEST_PLUGIN_NAME, "--upgrade"])
     assert result.exit_code == 0
 
 
-@github_xfail()
+@pytest.mark.pip
+@run_once
 def test_upgrade_failure(ape_plugins_runner):
     result = ape_plugins_runner.invoke(["install", "NOT_EXISTS", "--upgrade"])
     assert result.exit_code == 1
 
 
-@github_xfail()
+@pytest.mark.pip
+@run_once
 def test_install_multiple_in_one_str(ape_plugins_runner):
     result = ape_plugins_runner.invoke(["install", f"{TEST_PLUGIN_NAME} {TEST_PLUGIN_NAME_2}"])
     assert result.exit_code == 0
 
 
-@github_xfail()
+@pytest.mark.pip
+@run_once
 def test_install_from_config_file(ape_cli, runner, temp_config):
     plugins_config = {"plugins": [{"name": TEST_PLUGIN_NAME}]}
     with temp_config(plugins_config):
@@ -156,7 +160,8 @@ def test_install_from_config_file(ape_cli, runner, temp_config):
         assert TEST_PLUGIN_NAME in result.stdout
 
 
-@github_xfail()
+@pytest.mark.pip
+@run_once
 def test_uninstall(ape_cli, runner, installed_plugin):
     result = runner.invoke(
         ape_cli, ["plugins", "uninstall", TEST_PLUGIN_NAME, "--yes"], catch_exceptions=False