From 3fb4af2e4d0354a71fdea3a50d3fc3f9811c4551 Mon Sep 17 00:00:00 2001
From: Charlie <99198652+cdummett@users.noreply.github.com>
Date: Thu, 23 Mar 2023 17:48:43 +0000
Subject: [PATCH] feat: integration test for fuzzing

* feat: add fuzzing integration test

* fix: fix FileExistsError

* fix: correct args

* feat: reduce test steps
---
 tests/integration/test_fuzz.py                |  9 ++++
 vega_sim/scenario/fuzzed_markets/agents.py    |  5 +-
 .../scenario/fuzzed_markets/run_fuzz_test.py  | 53 ++++++++++---------
 vega_sim/scenario/fuzzed_markets/scenario.py  |  5 +-
 4 files changed, 45 insertions(+), 27 deletions(-)
 create mode 100644 tests/integration/test_fuzz.py

diff --git a/tests/integration/test_fuzz.py b/tests/integration/test_fuzz.py
new file mode 100644
index 000000000..f053860ff
--- /dev/null
+++ b/tests/integration/test_fuzz.py
@@ -0,0 +1,9 @@
+import pytest
+
+
+@pytest.mark.integration
+def test_fuzz_run():
+    # Simply testing that it doesn't error
+    import vega_sim.scenario.fuzzed_markets.run_fuzz_test as fuzz
+
+    fuzz._run(steps=50, output=True)
diff --git a/vega_sim/scenario/fuzzed_markets/agents.py b/vega_sim/scenario/fuzzed_markets/agents.py
index da60d7308..8ec78da13 100644
--- a/vega_sim/scenario/fuzzed_markets/agents.py
+++ b/vega_sim/scenario/fuzzed_markets/agents.py
@@ -246,7 +246,10 @@ def finalise(self):
                     range_color=range_color,
                 )
                 fig.update_traces(marker=dict(cornerradius=5))
-                os.mkdir("fuzz_plots")
+
+                if not os.path.exists("fuzz_plots"):
+                    os.mkdir("fuzz_plots")
+
                 fig.write_html("fuzz_plots/coverage.html")
                 fig.show()
 
diff --git a/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py b/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py
index 824624516..96f7e9371 100644
--- a/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py
+++ b/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py
@@ -11,27 +11,13 @@
 from vega_sim.tools.scenario_plots import fuzz_plots, plot_run_outputs
 
 
-def output_summary(output):
-    pass
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO)
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-s",
-        "--steps",
-        default=2 * 60 * 12,
-        type=int,
-    )
-    args = parser.parse_args()
-
+def _run(steps: int = 2880, output: bool = False):
     scenario = FuzzingScenario(
-        num_steps=args.steps,
+        num_steps=steps,
         step_length_seconds=30,
         block_length_seconds=1,
         transactions_per_block=4096,
+        output=output,
     )
 
     with VegaServiceNull(
@@ -44,15 +30,32 @@ def output_summary(output):
         scenario.run_iteration(
             vega=vega,
             network=Network.NULLCHAIN,
-            output_data=True,
+            output_data=output,
         )
 
-    os.mkdir("fuzz_plots")
+    if output:
+        if not os.path.exists("fuzz_plots"):
+            os.mkdir("fuzz_plots")
 
-    fuzz_figs = fuzz_plots()
-    for key, fig in fuzz_figs.items():
-        fig.savefig(f"fuzz_plots/fuzz-{key}.jpg")
+        fuzz_figs = fuzz_plots()
+        for key, fig in fuzz_figs.items():
+            fig.savefig(f"fuzz_plots/fuzz-{key}.jpg")
+
+        trading_figs = plot_run_outputs()
+        for key, fig in trading_figs.items():
+            fig.savefig(f"fuzz_plots/trading-{key}.jpg")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-s",
+        "--steps",
+        default=2 * 60 * 12,
+        type=int,
+    )
+    args = parser.parse_args()
 
-    trading_figs = plot_run_outputs()
-    for key, fig in trading_figs.items():
-        fig.savefig(f"fuzz_plots/trading-{key}.jpg")
+    _run(steps=args.steps, output=True)
diff --git a/vega_sim/scenario/fuzzed_markets/scenario.py b/vega_sim/scenario/fuzzed_markets/scenario.py
index 682cc887b..e8c92fb87 100644
--- a/vega_sim/scenario/fuzzed_markets/scenario.py
+++ b/vega_sim/scenario/fuzzed_markets/scenario.py
@@ -92,6 +92,7 @@ def __init__(
         n_markets: int = 2,
         step_length_seconds: Optional[float] = None,
         fuzz_market_config: Optional[dict] = None,
+        output: bool = True,
     ):
         super().__init__(
             state_extraction_fn=lambda vega, agents: state_extraction_fn(vega, agents),
@@ -113,6 +114,8 @@ def __init__(
         self.block_length_seconds = block_length_seconds
         self.transactions_per_block = transactions_per_block
 
+        self.output = output
+
     def configure_agents(
         self,
         vega: VegaServiceNull,
@@ -230,7 +233,7 @@ def configure_agents(
                         key_name=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}",
                         market_name=market_name,
                         asset_name=asset_name,
-                        output_plot_on_finalise=True,
+                        output_plot_on_finalise=self.output,
                         tag=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}",
                     )
                 )