diff --git a/.gitignore b/.gitignore index f0a798a83..65b46af6e 100644 --- a/.gitignore +++ b/.gitignore @@ -133,6 +133,7 @@ dmypy.json .vscode/* *.jpg +*.html vega_sim/bin/* rl_logs/ numerical_results/ diff --git a/poetry.lock b/poetry.lock index 64e4508bd..167f496ac 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. [[package]] name = "aiofiles" @@ -621,7 +621,6 @@ files = [ {file = "debugpy-1.6.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b5d1b13d7c7bf5d7cf700e33c0b8ddb7baf030fcf502f76fc061ddd9405d16c"}, {file = "debugpy-1.6.6-cp38-cp38-win32.whl", hash = "sha256:70ab53918fd907a3ade01909b3ed783287ede362c80c75f41e79596d5ccacd32"}, {file = "debugpy-1.6.6-cp38-cp38-win_amd64.whl", hash = "sha256:c05349890804d846eca32ce0623ab66c06f8800db881af7a876dc073ac1c2225"}, - {file = "debugpy-1.6.6-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:11a0f3a106f69901e4a9a5683ce943a7a5605696024134b522aa1bfda25b5fec"}, {file = "debugpy-1.6.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a771739902b1ae22a120dbbb6bd91b2cae6696c0e318b5007c5348519a4211c6"}, {file = "debugpy-1.6.6-cp39-cp39-win32.whl", hash = "sha256:549ae0cb2d34fc09d1675f9b01942499751d174381b6082279cf19cdb3c47cbe"}, {file = "debugpy-1.6.6-cp39-cp39-win_amd64.whl", hash = "sha256:de4a045fbf388e120bb6ec66501458d3134f4729faed26ff95de52a754abddb1"}, @@ -2188,6 +2187,13 @@ files = [ {file = "Pillow-9.4.0-1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:b8c2f6eb0df979ee99433d8b3f6d193d9590f735cf12274c108bd954e30ca858"}, {file = "Pillow-9.4.0-1-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:b70756ec9417c34e097f987b4d8c510975216ad26ba6e57ccb53bc758f490dab"}, {file = "Pillow-9.4.0-1-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:43521ce2c4b865d385e78579a082b6ad1166ebed2b1a2293c3be1d68dd7ca3b9"}, + {file = "Pillow-9.4.0-2-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:9d9a62576b68cd90f7075876f4e8444487db5eeea0e4df3ba298ee38a8d067b0"}, + {file = "Pillow-9.4.0-2-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:87708d78a14d56a990fbf4f9cb350b7d89ee8988705e58e39bdf4d82c149210f"}, + {file = "Pillow-9.4.0-2-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:8a2b5874d17e72dfb80d917213abd55d7e1ed2479f38f001f264f7ce7bae757c"}, + {file = "Pillow-9.4.0-2-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:83125753a60cfc8c412de5896d10a0a405e0bd88d0470ad82e0869ddf0cb3848"}, + {file = "Pillow-9.4.0-2-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:9e5f94742033898bfe84c93c831a6f552bb629448d4072dd312306bab3bd96f1"}, + {file = "Pillow-9.4.0-2-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:013016af6b3a12a2f40b704677f8b51f72cb007dac785a9933d5c86a72a7fe33"}, + {file = "Pillow-9.4.0-2-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:99d92d148dd03fd19d16175b6d355cc1b01faf80dae93c6c3eb4163709edc0a9"}, {file = "Pillow-9.4.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:2968c58feca624bb6c8502f9564dd187d0e1389964898f5e9e1fbc8533169157"}, {file = "Pillow-9.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c5c1362c14aee73f50143d74389b2c158707b4abce2cb055b7ad37ce60738d47"}, {file = "Pillow-9.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd752c5ff1b4a870b7661234694f24b1d2b9076b8bf337321a814c612665f343"}, @@ -2278,7 +2284,7 @@ name = "plotly" version = "5.13.0" description = "An open-source, interactive data visualization library for Python" category = "main" -optional = true +optional = false python-versions = ">=3.6" files = [ {file = "plotly-5.13.0-py2.py3-none-any.whl", hash = "sha256:4ac5db72176ce144f1fcde8d1ef7bdbccf5bb7a53e3d366b16fcd7c85319fdfd"}, @@ -2560,7 +2566,7 @@ files = [ cffi = ">=1.4.1" [package.extras] -docs = ["sphinx (>=1.6.5)", "sphinx_rtd_theme"] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] @@ -2664,9 +2670,7 @@ optional = true python-versions = "*" files = [ {file = "pytest-profiling-1.7.0.tar.gz", hash = "sha256:93938f147662225d2b8bd5af89587b979652426a8a6ffd7e73ec4a23e24b7f29"}, - {file = "pytest_profiling-1.7.0-py2.7.egg", hash = "sha256:3b255f9db36cb2dd7536a8e7e294c612c0be7f7850a7d30754878e4315d56600"}, {file = "pytest_profiling-1.7.0-py2.py3-none-any.whl", hash = "sha256:999cc9ac94f2e528e3f5d43465da277429984a1c237ae9818f8cfd0b06acb019"}, - {file = "pytest_profiling-1.7.0-py3.6.egg", hash = "sha256:6bce4e2edc04409d2f3158c16750fab8074f62d404cc38eeb075dff7fcbb996c"}, ] [package.dependencies] @@ -3209,7 +3213,7 @@ name = "tenacity" version = "8.2.1" description = "Retry code until it succeeds" category = "main" -optional = true +optional = false python-versions = ">=3.6" files = [ {file = "tenacity-8.2.1-py3-none-any.whl", hash = "sha256:dd1b769ca7002fda992322939feca5bee4fa11f39146b0af14e0b8d9f27ea854"}, @@ -3674,11 +3678,11 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [extras] agents = ["TA-Lib"] -jupyter = ["jupyterlab", "jupyter", "matplotlib", "plotly", "ipywidgets"] -learning = ["matplotlib", "tqdm", "torch"] -profile = ["snakeviz", "pytest-profiling"] +jupyter = ["ipywidgets", "jupyter", "jupyterlab", "matplotlib"] +learning = ["matplotlib", "torch", "tqdm"] +profile = ["pytest-profiling", "snakeviz"] [metadata] lock-version = "2.0" python-versions = "^3.9,<3.11" -content-hash = "03f33fc7317f25c107af181c8c8c1f334c271ce555dbc1b575c07210f7fd9609" +content-hash = "0dd61d25447a97fa8678284446f7607291208cf594b39d4440da70cd18034038" diff --git a/pyproject.toml b/pyproject.toml index 29c7e28d2..0118f170b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ snakeviz = {version = "^2.1.1", optional = true} pytest-profiling = {version = "^1.7.0", optional = true} ipywidgets = {version = "^7.7.1", optional = true} grpc-gateway-protoc-gen-openapiv2 = "^0.1.0" -plotly = {version = "^5.10.0", optional = true} +plotly = "^5.10.0" TA-Lib = {version = "^0.4.25", optional = true} python-dotenv = "^0.21.0" deprecated = "^1.2.13" @@ -46,12 +46,8 @@ pytest-xdist = "^2.5.0" requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" - - - - [tool.poetry.extras] learning = ["matplotlib", "tqdm", "torch"] -jupyter = ["jupyterlab", "jupyter", "matplotlib", "plotly", "ipywidgets"] +jupyter = ["jupyterlab", "jupyter", "matplotlib", "ipywidgets"] profile = ["snakeviz", "pytest-profiling"] agents = ["TA-Lib"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 4cb403dc4..2cbd50916 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -55,6 +55,7 @@ pexpect==4.8.0 ; python_version >= "3.9" and python_version < "3.11" and sys_pla pickleshare==0.7.5 ; python_version >= "3.9" and python_version < "3.11" pillow==9.4.0 ; python_version >= "3.9" and python_version < "3.11" platformdirs==3.0.0 ; python_version >= "3.9" and python_version < "3.11" +plotly==5.13.0 ; python_version >= "3.9" and python_version < "3.11" pluggy==1.0.0 ; python_version >= "3.9" and python_version < "3.11" prompt-toolkit==3.0.37 ; python_version >= "3.9" and python_version < "3.11" protobuf==3.20.3 ; python_version >= "3.9" and python_version < "3.11" @@ -84,6 +85,7 @@ scipy==1.10.1 ; python_version >= "3.9" and python_version < "3.11" setuptools==67.4.0 ; python_version >= "3.9" and python_version < "3.11" six==1.16.0 ; python_version >= "3.9" and python_version < "3.11" stack-data==0.6.2 ; python_version >= "3.9" and python_version < "3.11" +tenacity==8.2.1 ; python_version >= "3.9" and python_version < "3.11" toml==0.10.2 ; python_version >= "3.9" and python_version < "3.11" tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11" tornado==6.2 ; python_version >= "3.9" and python_version < "3.11" diff --git a/requirements-learning.txt b/requirements-learning.txt index 87bed9117..8dfc58ca4 100644 --- a/requirements-learning.txt +++ b/requirements-learning.txt @@ -55,6 +55,7 @@ pexpect==4.8.0 ; python_version >= "3.9" and python_version < "3.11" and sys_pla pickleshare==0.7.5 ; python_version >= "3.9" and python_version < "3.11" pillow==9.4.0 ; python_version >= "3.9" and python_version < "3.11" platformdirs==3.0.0 ; python_version >= "3.9" and python_version < "3.11" +plotly==5.13.0 ; python_version >= "3.9" and python_version < "3.11" pluggy==1.0.0 ; python_version >= "3.9" and python_version < "3.11" prompt-toolkit==3.0.37 ; python_version >= "3.9" and python_version < "3.11" protobuf==3.20.3 ; python_version >= "3.9" and python_version < "3.11" @@ -84,6 +85,7 @@ scipy==1.10.1 ; python_version >= "3.9" and python_version < "3.11" setuptools==67.4.0 ; python_version >= "3.9" and python_version < "3.11" six==1.16.0 ; python_version >= "3.9" and python_version < "3.11" stack-data==0.6.2 ; python_version >= "3.9" and python_version < "3.11" +tenacity==8.2.1 ; python_version >= "3.9" and python_version < "3.11" toml==0.10.2 ; python_version >= "3.9" and python_version < "3.11" tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11" torch==1.12.1 ; python_version >= "3.9" and python_version < "3.11" diff --git a/requirements.txt b/requirements.txt index d67ddc0b2..9e4096a51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ numpy==1.24.2 ; python_version >= "3.9" and python_version < "3.11" packaging==23.0 ; python_version >= "3.9" and python_version < "3.11" pandas==1.5.3 ; python_version >= "3.9" and python_version < "3.11" pillow==9.4.0 ; python_version >= "3.9" and python_version < "3.11" +plotly==5.13.0 ; python_version >= "3.9" and python_version < "3.11" protobuf==3.20.3 ; python_version >= "3.9" and python_version < "3.11" pycparser==2.21 ; python_version >= "3.9" and python_version < "3.11" pynacl==1.5.0 ; python_version >= "3.9" and python_version < "3.11" @@ -29,6 +30,7 @@ requests==2.28.2 ; python_version >= "3.9" and python_version < "3.11" scipy==1.10.1 ; python_version >= "3.9" and python_version < "3.11" setuptools==67.4.0 ; python_version >= "3.9" and python_version < "3.11" six==1.16.0 ; python_version >= "3.9" and python_version < "3.11" +tenacity==8.2.1 ; python_version >= "3.9" and python_version < "3.11" toml==0.10.2 ; python_version >= "3.9" and python_version < "3.11" urllib3==1.26.14 ; python_version >= "3.9" and python_version < "3.11" wrapt==1.14.1 ; python_version >= "3.9" and python_version < "3.11" diff --git a/scripts/run-docker-fuzz-test.sh b/scripts/run-docker-fuzz-test.sh index 95b148bc2..afbadbd44 100755 --- a/scripts/run-docker-fuzz-test.sh +++ b/scripts/run-docker-fuzz-test.sh @@ -10,4 +10,4 @@ docker run \ --platform linux/amd64 \ -v "${RESULT_DIR}:/tmp" \ vega_sim_learning:latest \ - python -m vega_sim.scenario.adhoc -s fuzz_test + python -m vega_sim.scenario.fuzzed_markets.run_fuzz_test --steps 2*60*24 diff --git a/tests/integration/test_plot_gen.py b/tests/integration/test_plot_gen.py index e41901d36..40eadfeb4 100644 --- a/tests/integration/test_plot_gen.py +++ b/tests/integration/test_plot_gen.py @@ -30,10 +30,14 @@ def generate_trading_plot(): opening_auction_trade_amount=0.0001, market_order_trader_base_order_size=0.01, ) - with VegaServiceNull(warn_on_raw_data_access=False, retain_log_files=True) as vega: + with VegaServiceNull( + warn_on_raw_data_access=False, + retain_log_files=True, + ) as vega: scen.run_iteration(vega=vega, output_data=True) - plot = plot_run_outputs() - plot.savefig("run.jpg") + figs = plot_run_outputs() + for key, value in figs.items(): + value.savefig(f"run.jpg") if __name__ == "__main__": diff --git a/vega_sim/scenario/fuzzed_markets/agents.py b/vega_sim/scenario/fuzzed_markets/agents.py index 5b9df7168..baf25b2fd 100644 --- a/vega_sim/scenario/fuzzed_markets/agents.py +++ b/vega_sim/scenario/fuzzed_markets/agents.py @@ -1,15 +1,31 @@ from vega_sim.environment.agent import StateAgentWithWallet from typing import Optional from vega_sim.null_service import VegaServiceNull -from numpy import array from numpy.random import RandomState from vega_sim.proto.vega import markets as markets_protos import vega_sim.proto.vega as vega_protos +import pandas as pd + class FuzzingAgent(StateAgentWithWallet): NAME_BASE = "fuzzing_agent" + # Set the memory which all instances can modify + MEMORY = { + key: [] + for key in ( + "TRADING_MODE", + "COMMAND", + "TYPE", + "SIDE", + "TIME_IN_FORCE", + "PEGGED_REFERENCE", + ) + } + # Set the output flag which all instances can modify + OUTPUTTED = False + def __init__( self, key_name: str, @@ -20,6 +36,7 @@ def __init__( state_update_freq: Optional[int] = None, initial_asset_mint: float = 1e9, random_state: Optional[RandomState] = None, + output_plot_on_finalise: bool = False, ): super().__init__( key_name=key_name, @@ -32,6 +49,7 @@ def __init__( self.asset_name = asset_name self.initial_asset_mint = initial_asset_mint self.random_state = random_state if random_state is not None else RandomState() + self.output_plot_on_finalise = output_plot_on_finalise def initialise( self, vega: VegaServiceNull, create_key: bool = True, mint_key: bool = True @@ -63,9 +81,9 @@ def step(self, vega_state): self.curr_price = vega_state.market_state[self.market_id].midprice - submissions = [self.create_fuzzed_submission() for _ in range(20)] - amendments = [self.create_fuzzed_amendment() for _ in range(10)] - cancellations = [self.create_fuzzed_cancellation() for _ in range(1)] + submissions = [self.create_fuzzed_submission(vega_state) for _ in range(20)] + amendments = [self.create_fuzzed_amendment(vega_state) for _ in range(10)] + cancellations = [self.create_fuzzed_cancellation(vega_state) for _ in range(1)] self.vega.submit_instructions( key_name=self.key_name, @@ -75,7 +93,7 @@ def step(self, vega_state): cancellations=cancellations, ) - def create_fuzzed_cancellation(self): + def create_fuzzed_cancellation(self, vega_state): order_id = self._select_order_id() return self.vega.create_order_cancellation( @@ -83,7 +101,7 @@ def create_fuzzed_cancellation(self): market_id=self.market_id, ) - def create_fuzzed_amendment(self): + def create_fuzzed_amendment(self, vega_state): order_id = self._select_order_id() return self.vega.create_order_amendment( @@ -123,17 +141,33 @@ def create_fuzzed_amendment(self): pegged_offset=self.random_state.normal(loc=0, scale=10), ) - def create_fuzzed_submission(self): - return self.vega.create_order_submission( - market_id=self.market_id, - side=self.random_state.choice( - a=["SIDE_UNSPECIFIED", "SIDE_BUY", "SIDE_SELL"], - ), - size=self.random_state.poisson(lam=10), - order_type=self.random_state.choice( - a=["TYPE_UNSPECIFIED", "TYPE_MARKET", "TYPE_LIMIT"] - ), - time_in_force=self.random_state.choice( + def create_fuzzed_submission(self, vega_state): + FuzzingAgent.MEMORY["TRADING_MODE"].append( + markets_protos.Market.TradingMode.Name( + vega_state.market_state[self.market_id].trading_mode + ) + ) + FuzzingAgent.MEMORY["COMMAND"].append("ORDER_SUBMISSION") + FuzzingAgent.MEMORY["SIDE"].append( + self.random_state.choice( + a=[ + "SIDE_UNSPECIFIED", + "SIDE_BUY", + "SIDE_SELL", + ] + ) + ) + FuzzingAgent.MEMORY["TYPE"].append( + self.random_state.choice( + a=[ + "TYPE_UNSPECIFIED", + "TYPE_MARKET", + "TYPE_LIMIT", + ] + ) + ) + FuzzingAgent.MEMORY["TIME_IN_FORCE"].append( + self.random_state.choice( a=[ "TIME_IN_FORCE_UNSPECIFIED", "TIME_IN_FORCE_GTC", @@ -143,7 +177,26 @@ def create_fuzzed_submission(self): "TIME_IN_FORCE_FOK", "TIME_IN_FORCE_IOC", ] - ), + ) + ) + FuzzingAgent.MEMORY["PEGGED_REFERENCE"].append( + self.random_state.choice( + a=[ + "PEGGED_REFERENCE_UNSPECIFIED", + "PEGGED_REFERENCE_MID", + "PEGGED_REFERENCE_BEST_BID", + "PEGGED_REFERENCE_BEST_ASK", + ], + p=[0.5, 0.5 / 3, 0.5 / 3, 0.5 / 3], + ) + ) + + return self.vega.create_order_submission( + market_id=self.market_id, + side=FuzzingAgent.MEMORY["SIDE"][-1], + size=self.random_state.poisson(lam=10), + order_type=FuzzingAgent.MEMORY["TYPE"][-1], + time_in_force=FuzzingAgent.MEMORY["TIME_IN_FORCE"][-1], price=self.random_state.choice( a=[None, self.random_state.normal(loc=self.curr_price, scale=10)] ), @@ -154,14 +207,7 @@ def create_fuzzed_submission(self): ) * 1e9 ), - pegged_reference=self.random_state.choice( - a=[ - "PEGGED_REFERENCE_UNSPECIFIED", - "PEGGED_REFERENCE_MID", - "PEGGED_REFERENCE_BEST_BID", - "PEGGED_REFERENCE_BEST_ASK", - ] - ), + pegged_reference=FuzzingAgent.MEMORY["PEGGED_REFERENCE"][-1], pegged_offset=self.random_state.normal(loc=0, scale=10), ) @@ -173,6 +219,41 @@ def _select_order_id(self): else: return None + def finalise(self): + if self.output_plot_on_finalise: + if not FuzzingAgent.OUTPUTTED: + import plotly.express as px + + FuzzingAgent.OUTPUTTED = True + df = pd.DataFrame.from_dict(FuzzingAgent.MEMORY) + df = ( + df.groupby(list(FuzzingAgent.MEMORY.keys())) + .size() + .reset_index() + .rename(columns={0: "count"}) + ) + + range_color = (10, 5000) + custom_color_scale = [ + [0, "red"], + [range_color[0] / range_color[1], "red"], + [range_color[0] / range_color[1], "yellow"], + [1, "green"], + ] + + fig = px.treemap( + df, + title="Fuzzed Trader Coverage", + path=list(self.__class__.MEMORY.keys()), + values="count", + color="count", + color_continuous_scale=custom_color_scale, + range_color=range_color, + ) + fig.update_traces(marker=dict(cornerradius=5)) + fig.write_html("coverage.html") + fig.show() + class DegenerateTrader(StateAgentWithWallet): NAME_BASE = "degenerate_trader" @@ -202,6 +283,8 @@ def __init__( self.initial_asset_mint = initial_asset_mint self.step_bias = step_bias + self.close_outs = 0 + def initialise(self, vega: VegaServiceNull, create_key: bool = True, mint_key=True): super().initialise(vega, create_key) @@ -221,16 +304,6 @@ def initialise(self, vega: VegaServiceNull, create_key: bool = True, mint_key=Tr self.vega.wait_fn(5) def step(self, vega_state): - if ( - vega_state.market_state[self.market_id].trading_mode - != markets_protos.Market.TradingMode.TRADING_MODE_CONTINUOUS - ): - return - if self.random_state.rand() > self.step_bias: - return - - midprice = vega_state.market_state[self.market_id].midprice - account = self.vega.party_account( key_name=self.key_name, wallet_name=self.wallet_name, @@ -245,8 +318,19 @@ def step(self, vega_state): amount=self.initial_asset_mint, asset=self.asset_id, ) + self.close_outs += 1 + return + + if ( + vega_state.market_state[self.market_id].trading_mode + != markets_protos.Market.TradingMode.TRADING_MODE_CONTINUOUS + ): + return + if self.random_state.rand() > self.step_bias: return + midprice = vega_state.market_state[self.market_id].midprice + if account.general > 0: add_to_margin = max( (account.general + account.margin) * self.size_factor - account.margin, @@ -298,6 +382,8 @@ def __init__( self.commitment_amount = 0 + self.close_outs = 0 + def initialise(self, vega: VegaServiceNull, create_key: bool = True, mint_key=True): super().initialise(vega, create_key) @@ -317,9 +403,6 @@ def initialise(self, vega: VegaServiceNull, create_key: bool = True, mint_key=Tr self.vega.wait_fn(5) def step(self, vega_state): - if self.random_state.rand() > self.step_bias: - return - account = self.vega.party_account( key_name=self.key_name, wallet_name=self.wallet_name, @@ -336,6 +419,10 @@ def step(self, vega_state): amount=self.initial_asset_mint, asset=self.asset_id, ) + self.close_outs += 1 + return + + if self.random_state.rand() > self.step_bias: return if self.commitment_amount < self.commitment_factor * (total_balance): diff --git a/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py b/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py new file mode 100644 index 000000000..f6c4cee9d --- /dev/null +++ b/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py @@ -0,0 +1,54 @@ +import logging +import argparse + +from vega_sim.null_service import VegaServiceNull + +from vega_sim.scenario.constants import Network +from vega_sim.scenario.fuzzed_markets.scenario import FuzzingScenario + +from vega_sim.tools.scenario_plots import fuzz_plots, plot_run_outputs + + +def output_summary(output): + pass + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument( + "-s", + "--steps", + default=2 * 60 * 12, + type=int, + ) + args = parser.parse_args() + + scenario = FuzzingScenario( + num_steps=args.steps, + step_length_seconds=30, + block_length_seconds=1, + transactions_per_block=4096, + ) + + with VegaServiceNull( + warn_on_raw_data_access=False, + seconds_per_block=scenario.block_length_seconds, + transactions_per_block=scenario.transactions_per_block, + retain_log_files=True, + use_full_vega_wallet=False, + ) as vega: + scenario.run_iteration( + vega=vega, + network=Network.NULLCHAIN, + output_data=True, + ) + + fuzz_figs = fuzz_plots() + for key, fig in fuzz_figs.items(): + fig.savefig(f"fuzz-{key}.jpg") + + trading_figs = plot_run_outputs() + for key, fig in trading_figs.items(): + fig.savefig(f"trading-{key}.jpg") diff --git a/vega_sim/scenario/fuzzed_markets/scenario.py b/vega_sim/scenario/fuzzed_markets/scenario.py index e5a9565a8..682cc887b 100644 --- a/vega_sim/scenario/fuzzed_markets/scenario.py +++ b/vega_sim/scenario/fuzzed_markets/scenario.py @@ -24,6 +24,64 @@ FuzzyLiquidityProvider, ) +import datetime +from typing import Optional, Dict +from dataclasses import dataclass +from vega_sim.scenario.common.agents import ExponentialShapedMarketMaker +import pandas as pd + + +@dataclass +class MarketHistoryAdditionalData: + at_time: datetime.datetime + external_prices: Dict[str, float] + trader_close_outs: Dict[str, int] + liquidity_provider_close_outs: Dict[str, int] + + +def state_extraction_fn(vega: VegaServiceNull, agents: dict): + at_time = vega.get_blockchain_time() + + external_prices = {} + trader_close_outs = {} + liquidity_provider_close_outs = {} + + for _, agent in agents.items(): + if isinstance(agent, ExponentialShapedMarketMaker): + external_prices[agent.market_id] = agent.curr_price + if isinstance(agent, DegenerateTrader): + trader_close_outs[agent.market_id] = ( + trader_close_outs.get(agent.market_id, 0) + agent.close_outs + ) + if isinstance(agent, DegenerateLiquidityProvider): + liquidity_provider_close_outs[agent.market_id] = ( + liquidity_provider_close_outs.get(agent.market_id, 0) + agent.close_outs + ) + + return MarketHistoryAdditionalData( + at_time=at_time, + external_prices=external_prices, + trader_close_outs=trader_close_outs, + liquidity_provider_close_outs=liquidity_provider_close_outs, + ) + + +def additional_data_to_rows(data) -> List[pd.Series]: + results = [] + for market_id in data.external_prices.keys(): + results.append( + { + "time": data.at_time, + "market_id": market_id, + "external_price": data.external_prices.get(market_id, np.NaN), + "trader_close_outs": data.trader_close_outs.get(market_id, 0), + "liquidity_provider_close_outs": data.liquidity_provider_close_outs.get( + market_id, 0 + ), + } + ) + return results + class FuzzingScenario(Scenario): def __init__( @@ -31,11 +89,16 @@ def __init__( num_steps: int = 60 * 24 * 30 * 3, transactions_per_block: int = 4096, block_length_seconds: float = 1, - n_markets: int = 5, + n_markets: int = 2, step_length_seconds: Optional[float] = None, fuzz_market_config: Optional[dict] = None, ): - super().__init__() + super().__init__( + state_extraction_fn=lambda vega, agents: state_extraction_fn(vega, agents), + additional_data_output_fns={ + "additional_data.csv": lambda data: additional_data_to_rows(data), + }, + ) self.n_markets = n_markets self.fuzz_market_config = fuzz_market_config @@ -87,7 +150,7 @@ def configure_agents( # Create fuzzed price process price_process = random_walk( num_steps=self.num_steps + 1, - sigma=random_state.rand(), + sigma=random_state.rand() * 1e1, drift=random_state.rand() * 1e-3, starting_price=1000, decimal_precision=int(market_config.decimal_places), @@ -167,6 +230,7 @@ def configure_agents( key_name=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}", market_name=market_name, asset_name=asset_name, + output_plot_on_finalise=True, tag=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}", ) ) @@ -180,8 +244,9 @@ def configure_agents( market_name=market_name, asset_name=asset_name, side=side, - initial_asset_mint=5_000, + initial_asset_mint=1_000, size_factor=0.7, + step_bias=0.01, tag=f"MARKET_{str(i_market).zfill(3)}_SIDE_{side}_AGENT_{str(i_agent).zfill(3)}", ) ) @@ -193,8 +258,9 @@ def configure_agents( key_name=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}", market_name=market_name, asset_name=asset_name, - initial_asset_mint=5_000, + initial_asset_mint=1_000, commitment_factor=0.7, + step_bias=0.01, tag=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}", ) ) diff --git a/vega_sim/scenario/registry.py b/vega_sim/scenario/registry.py index 667c12ad1..1243c0784 100644 --- a/vega_sim/scenario/registry.py +++ b/vega_sim/scenario/registry.py @@ -199,7 +199,8 @@ ), "parameter_experiment": lambda: ParameterExperiment(), "fuzz_test": lambda: FuzzingScenario( - num_steps=60 * 60 * 3, + num_steps=2 * 60 * 12, + step_length_seconds=30, block_length_seconds=1, transactions_per_block=4096, ), diff --git a/vega_sim/scenario/scenario.py b/vega_sim/scenario/scenario.py index a8bc5be4a..4aa817731 100644 --- a/vega_sim/scenario/scenario.py +++ b/vega_sim/scenario/scenario.py @@ -15,10 +15,12 @@ def __init__( state_extraction_fn: Optional[ Callable[[VegaService, Dict[str, Agent]], Any] ] = None, + additional_data_output_fns: Optional[Dict[str, Callable]] = None, ): self.agents = [] self.env: Optional[MarketEnvironment] = None self.state_extraction_fn = state_extraction_fn + self.additional_data_output_fns = additional_data_output_fns @abc.abstractmethod def configure_agents( @@ -82,6 +84,11 @@ def run_iteration( ) if output_data: market_data_standard_output(self.get_run_data()) + if self.additional_data_output_fns is not None: + market_data_standard_output( + self.get_additional_run_data(), + custom_output_fns=self.additional_data_output_fns, + ) return outputs diff --git a/vega_sim/tools/scenario_output.py b/vega_sim/tools/scenario_output.py index 25f10d066..de206a0df 100644 --- a/vega_sim/tools/scenario_output.py +++ b/vega_sim/tools/scenario_output.py @@ -12,6 +12,7 @@ ORDER_BOOK_FILE_NAME = "depth_data.csv" TRADES_FILE_NAME = "trades.csv" ACCOUNTS_FILE_NAME = "accounts.csv" +FUZZING_FILE_NAME = "additional_data.csv" def history_data_to_row(data: MarketHistoryData) -> List[pd.Series]: @@ -24,6 +25,7 @@ def history_data_to_row(data: MarketHistoryData) -> List[pd.Series]: "time": data.at_time, "mark_price": market_data.mark_price, "market_id": market_id, + "mark_price": market_data.mark_price, "open_interest": market_data.open_interest, "best_bid": market_data.best_bid_price, "best_offer": market_data.best_offer_price, @@ -200,3 +202,15 @@ def load_accounts_df( df["time"] = pd.to_datetime(df.time * 1e9) df = df.set_index("time") return df + + +def load_fuzzing_df( + run_name: Optional[str] = None, + output_path: str = DEFAULT_PATH, +) -> pd.DataFrame: + run_name = run_name if run_name is not None else DEFAULT_RUN_NAME + df = pd.read_csv(os.path.join(output_path, run_name, FUZZING_FILE_NAME)) + if not df.empty: + df["time"] = pd.to_datetime(df.time * 1e9) + df = df.set_index("time") + return df diff --git a/vega_sim/tools/scenario_plots.py b/vega_sim/tools/scenario_plots.py index 62c72ca8b..cc5cf2593 100644 --- a/vega_sim/tools/scenario_plots.py +++ b/vega_sim/tools/scenario_plots.py @@ -6,6 +6,22 @@ from matplotlib.axes import Axes from matplotlib.figure import Figure from matplotlib.markers import MarkerStyle +from matplotlib.gridspec import GridSpec, SubplotSpec, GridSpecFromSubplotSpec + +from vega_sim.proto.vega import markets + +import numpy as np + + +TRADING_MODE_COLORS = { + 0: (200 / 255, 200 / 255, 200 / 255), + 1: (204 / 255, 255 / 255, 15 / 2553), + 2: (255 / 255, 204 / 255, 153 / 255), + 3: (153 / 255, 204 / 255, 244 / 255), + 4: (255 / 255, 133 / 255, 133 / 255), + 5: (255 / 255, 204 / 255, 255 / 255), +} + """ Thoughts for plots @@ -18,6 +34,7 @@ load_market_data_df, load_order_book_df, load_trades_df, + load_fuzzing_df, ) @@ -174,35 +191,327 @@ def plot_target_stake(ax: Axes, market_data_df: pd.DataFrame) -> None: ax.plot(market_data_df["target_stake"]) -def plot_run_outputs(run_name: Optional[str] = None) -> Figure: +def plot_run_outputs(run_name: Optional[str] = None) -> list[Figure]: order_df = load_order_book_df(run_name=run_name) trades_df = load_trades_df(run_name=run_name) data_df = load_market_data_df(run_name=run_name) accounts_df = load_accounts_df(run_name=run_name) - mid_df = order_df[order_df.level == 0].groupby("time")[["price"]].sum() / 2 + figs = {} + + for market_id in data_df["market_id"].unique(): + market_order_df = order_df[order_df["market_id"] == market_id] + market_trades_df = trades_df[trades_df["market_id"] == market_id] + market_data_df = data_df[data_df["market_id"] == market_id] + market_accounts_df = accounts_df[accounts_df["market_id"] == market_id] + + market_mid_df = ( + market_order_df[market_order_df.level == 0].groupby("time")[["price"]].sum() + / 2 + ) + + fig = plt.figure(layout="tight", figsize=(8, 10)) + + ax = plt.subplot(411) + ax2 = plt.subplot(423) + ax3 = plt.subplot(424) + ax4 = plt.subplot(425) + ax5 = plt.subplot(426) + ax6 = plt.subplot(427) + ax7 = plt.subplot(428) + + plot_trading_summary(ax, market_trades_df, market_order_df, market_mid_df) + plot_total_traded_volume(ax2, market_trades_df) + plot_spread(ax3, order_book_df=market_order_df) + plot_open_interest(ax4, market_data_df) + plot_open_notional(ax5, market_data_df=market_data_df, price_df=market_mid_df) + plot_margin_totals(ax6, accounts_df=market_accounts_df) + plot_target_stake(ax7, market_data_df=market_data_df) + + figs[market_id] = fig + + return figs + + +def plot_trading_mode( + fig: Figure, data_df: pd.DataFrame, ss: Optional[SubplotSpec] = None +): + """Plots the proportion of time spent in different trading modes of a market. + + Args: + fig (Figure): + The Figure object to which the subplots will be added. + data_df (pd.DataFrame): + The DataFrame containing market data, including the trading mode of the + market at each time step. + gs (Optional[SubplotSpec]): + An optional SubplotSpec object defining the placement of the subplots. If + not specified, a default GridSpec with two rows and one column will be used. + + The function adds two subplots to the Figure object fig. The top subplot shows a + step plot with a fill between each step representing the trading mode at the current + time step. The bottom subplot shows a stacked area plot of the same information, + with each area representing the proportion of time spent in a particular trading + mode. The legend in the bottom subplot shows the name of each trading mode. + + TRADING_MODE_COLORS is a dictionary that maps each trading mode to a color used in + the plots. + """ + + if ss is None: + gs = GridSpec( + nrows=2, + ncols=1, + height_ratios=[1, 5], + hspace=0.1, + ) + else: + gs = GridSpecFromSubplotSpec( + subplot_spec=ss, + nrows=2, + ncols=1, + height_ratios=[1, 5], + hspace=0.1, + ) + ax0 = fig.add_subplot(gs[0, 0]) + ax1 = fig.add_subplot(gs[1, 0]) + + ax0.set_title( + "Trading Mode Analysis", loc="left", fontsize=12, color=(0.3, 0.3, 0.3) + ) + names = [] + for name, value in markets.Market.TradingMode.items(): + names.append(name) + series = (data_df["market_trading_mode"] == value).astype(int) + ax0.fill_between( + series.index, + series, + step="post", + alpha=1, + color=TRADING_MODE_COLORS[value], + linewidth=0, + ) + + data_df = data_df.merge( + (data_df["market_trading_mode"] == value).cumsum().rename(name), + left_index=True, + right_index=True, + ) + data_df = data_df[names].divide(data_df[names].sum(axis=1), axis=0) + + ax1.stackplot( + data_df.index, + *[data_df[name].values for name in names], + colors=TRADING_MODE_COLORS.values(), + ) + + ax0.get_xaxis().set_visible(False) + ax0.get_yaxis().set_visible(False) + + ax1.set_ylabel("PROPORTION OF TIME IN MODE") + ax1.legend(labels=names, loc="lower right") + + +def plot_price_comparison( + fig: Figure, + data_df: pd.DataFrame, + fuzzing_df: pd.DataFrame, + ss: Optional[SubplotSpec], +): + """Plots the external price and mark price along with their respective volatilities. + + Args: + fig (Figure): + Figure object to plot the data on. + fuzzing_df (pd.DataFrame): + DataFrame containing fuzzing data. + data_df (pd.DataFrame): + DataFrame containing market data. + ss (Optional[SubplotSpec]): + SubplotSpec object representing the position of the subplot on the figure. + """ + if ss is None: + gs = GridSpec( + nrows=1, + ncols=1, + ) + else: + gs = GridSpecFromSubplotSpec( + subplot_spec=ss, + nrows=1, + ncols=1, + ) + ax0 = fig.add_subplot(gs[0, 0]) + + ax0.set_title("Price Analysis", loc="left", fontsize=12, color=(0.3, 0.3, 0.3)) + + external_price_series = fuzzing_df["external_price"].replace(0, np.nan) + mark_price_series = data_df["mark_price"].replace(0, np.nan) + + ax0.plot(external_price_series) + ax0.plot(mark_price_series) + + ep_volatility = external_price_series.var() / external_price_series.size + mp_volatility = mark_price_series.var() / mark_price_series.size + + ax0.text( + x=0.1, + y=0.1, + s=f"external-price volatility = {round(ep_volatility, 1)}\nmark-price volatility = {round(mp_volatility, 1)}", + fontsize=8, + bbox=dict(facecolor="white", alpha=1), + transform=ax0.transAxes, + ) + + ax0.set_ylabel("PRICE") + ax0.legend(labels=["external price", "mark price"]) + + +def plot_degen_close_outs( + fig: Figure, + accounts_df: pd.DataFrame, + fuzzing_df: pd.DataFrame, + ss: Optional[SubplotSpec] = None, +): + """Plots the number of close outs of degen traders and degen liquidity providers. + + Args: + fig (matplotlib.figure.Figure): + The figure object to plot onto. + accounts_df (pandas.DataFrame): + A dataframe containing the accounts data. + fuzzing_df (pandas.DataFrame): + A dataframe containing the fuzzing data. + ss (Optional[matplotlib.gridspec.SubplotSpec]): + A subplot specification for the plot. Default is None. + """ + if ss is None: + gs = GridSpec( + nrows=2, + ncols=1, + height_ratios=[1, 1], + hspace=0.15, + ) + else: + gs = GridSpecFromSubplotSpec( + subplot_spec=ss, + nrows=2, + ncols=1, + height_ratios=[1, 1], + hspace=0.15, + ) + + ax0 = fig.add_subplot(gs[0, 0]) + ax1 = fig.add_subplot(gs[1, 0]) + + ax0.set_title("Close Out Analysis", loc="left", fontsize=12, color=(0.3, 0.3, 0.3)) + + insurance_pool_ds = accounts_df["balance"][ + (accounts_df["party_id"] == "network") & (accounts_df["type"] == 1) + ] + trader_close_outs_ds = fuzzing_df["trader_close_outs"] + liquidity_provider_close_outs_ds = fuzzing_df["liquidity_provider_close_outs"] + + ax0r = ax0.twinx() + + ln0 = ax0r.plot( + insurance_pool_ds.index, + insurance_pool_ds.values, + "b.-", + markersize=1, + label="insurance pool", + ) + ln1 = ax0.plot( + trader_close_outs_ds.index, + trader_close_outs_ds.values, + "r.-", + markersize=1, + label="degen trader close outs", + ) + + lns = ln0 + ln1 + ax0.legend(handles=lns, labels=[ln.get_label() for ln in lns], loc="upper left") + + ax0.set_ylabel("NB CLOSE OUTS") + ax0r.set_ylabel("INSURANCE POOL", position="right") + ax0r.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) + + ax0.get_xaxis().set_visible(False) + + ax1r = ax1.twinx() + + ln3 = ax1r.plot( + insurance_pool_ds.index, + insurance_pool_ds.values, + "b.-", + markersize=1, + label="insurance pool", + ) + ln4 = ax1.plot( + liquidity_provider_close_outs_ds.index, + liquidity_provider_close_outs_ds.values, + "r.-", + markersize=1, + label="degen LP close outs", + ) + + lns = ln3 + ln4 + ax1.legend(handles=lns, labels=[ln.get_label() for ln in lns], loc="upper left") + + ax1.set_ylabel("NB CLOSE OUTS") + ax1r.set_ylabel("INSURANCE POOL", position="right") + ax1r.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) + + +def fuzz_plots(run_name: Optional[str] = None) -> Figure: + data_df = load_market_data_df(run_name=run_name) + accounts_df = load_accounts_df(run_name=run_name) + fuzzing_df = load_fuzzing_df(run_name=run_name) + + markets = data_df.market_id.unique() + + figs = {} + for market_id in markets: + market_data_df = data_df[data_df["market_id"] == market_id] + market_accounts_df = accounts_df[accounts_df["market_id"] == market_id] + market_fuzzing_df = fuzzing_df[fuzzing_df["market_id"] == market_id] + + fig = plt.figure(figsize=[8, 10]) + fig.suptitle( + f"Fuzz Testing Plots", + fontsize=18, + fontweight="bold", + color=(0.2, 0.2, 0.2), + ) + fig.tight_layout() + + plt.rcParams.update({"font.size": 8}) - fig = plt.figure(layout="tight", figsize=(8, 10)) + gs = GridSpec(nrows=3, ncols=1, height_ratios=[2, 2, 3], hspace=0.3) - ax = plt.subplot(411) - ax2 = plt.subplot(423) - ax3 = plt.subplot(424) - ax4 = plt.subplot(425) - ax5 = plt.subplot(426) - ax6 = plt.subplot(427) - ax7 = plt.subplot(428) + plot_trading_mode(fig, ss=gs[0, 0], data_df=market_data_df) + plot_price_comparison( + fig, + ss=gs[1, 0], + data_df=market_data_df, + fuzzing_df=market_fuzzing_df, + ) + plot_degen_close_outs( + fig, + ss=gs[2, 0], + accounts_df=market_accounts_df, + fuzzing_df=market_fuzzing_df, + ) - plot_trading_summary(ax, trades_df, order_df, mid_df) - plot_total_traded_volume(ax2, trades_df) - plot_spread(ax3, order_book_df=order_df) - plot_open_interest(ax4, data_df) - plot_open_notional(ax5, market_data_df=data_df, price_df=mid_df) - plot_margin_totals(ax6, accounts_df=accounts_df) - plot_target_stake(ax7, market_data_df=data_df) + figs[market_id] = fig - return fig + return figs if __name__ == "__main__": - fig = plot_run_outputs() - fig.savefig("output.jpg") + figs = fuzz_plots() + for key, fig in figs.items(): + fig.savefig(f"fuzz-{key}.jpg") + figs = plot_run_outputs() + for key, fig in figs.items(): + fig.savefig(f"rl-{key}.jpg")