feat: integration test for fuzzing

* feat: add fuzzing integration test

* fix: fix FileExistsError

* fix: correct args

* feat: reduce test steps

tests/integration/test_fuzz.py

@@ -0,0 +1,9 @@
+import pytest
+
+
+@pytest.mark.integration
+def test_fuzz_run():
+    # Simply testing that it doesn't error
+    import vega_sim.scenario.fuzzed_markets.run_fuzz_test as fuzz
+
+    fuzz._run(steps=50, output=True)

vega_sim/scenario/fuzzed_markets/agents.py

@@ -246,7 +246,10 @@ def finalise(self):
                     range_color=range_color,
                 )
                 fig.update_traces(marker=dict(cornerradius=5))
-                os.mkdir("fuzz_plots")
+
+                if not os.path.exists("fuzz_plots"):
+                    os.mkdir("fuzz_plots")
+
                 fig.write_html("fuzz_plots/coverage.html")
                 fig.show()
 

vega_sim/scenario/fuzzed_markets/run_fuzz_test.py

@@ -11,27 +11,13 @@
 from vega_sim.tools.scenario_plots import fuzz_plots, plot_run_outputs
 
 
-def output_summary(output):
-    pass
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO)
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-s",
-        "--steps",
-        default=2 * 60 * 12,
-        type=int,
-    )
-    args = parser.parse_args()
-
+def _run(steps: int = 2880, output: bool = False):
     scenario = FuzzingScenario(
-        num_steps=args.steps,
+        num_steps=steps,
         step_length_seconds=30,
         block_length_seconds=1,
         transactions_per_block=4096,
+        output=output,
     )
 
     with VegaServiceNull(
@@ -44,15 +30,32 @@ def output_summary(output):
         scenario.run_iteration(
             vega=vega,
             network=Network.NULLCHAIN,
-            output_data=True,
+            output_data=output,
         )
 
-    os.mkdir("fuzz_plots")
+    if output:
+        if not os.path.exists("fuzz_plots"):
+            os.mkdir("fuzz_plots")
 
-    fuzz_figs = fuzz_plots()
-    for key, fig in fuzz_figs.items():
-        fig.savefig(f"fuzz_plots/fuzz-{key}.jpg")
+        fuzz_figs = fuzz_plots()
+        for key, fig in fuzz_figs.items():
+            fig.savefig(f"fuzz_plots/fuzz-{key}.jpg")
+
+        trading_figs = plot_run_outputs()
+        for key, fig in trading_figs.items():
+            fig.savefig(f"fuzz_plots/trading-{key}.jpg")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-s",
+        "--steps",
+        default=2 * 60 * 12,
+        type=int,
+    )
+    args = parser.parse_args()
 
-    trading_figs = plot_run_outputs()
-    for key, fig in trading_figs.items():
-        fig.savefig(f"fuzz_plots/trading-{key}.jpg")
+    _run(steps=args.steps, output=True)

vega_sim/scenario/fuzzed_markets/scenario.py

@@ -92,6 +92,7 @@ def __init__(
         n_markets: int = 2,
         step_length_seconds: Optional[float] = None,
         fuzz_market_config: Optional[dict] = None,
+        output: bool = True,
     ):
         super().__init__(
             state_extraction_fn=lambda vega, agents: state_extraction_fn(vega, agents),
@@ -113,6 +114,8 @@ def __init__(
         self.block_length_seconds = block_length_seconds
         self.transactions_per_block = transactions_per_block
 
+        self.output = output
+
     def configure_agents(
         self,
         vega: VegaServiceNull,
@@ -230,7 +233,7 @@ def configure_agents(
                         key_name=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}",
                         market_name=market_name,
                         asset_name=asset_name,
-                        output_plot_on_finalise=True,
+                        output_plot_on_finalise=self.output,
                         tag=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}",
                     )
                 )