diff --git a/.env b/.env index 1fae73b01..bdc911bac 100644 --- a/.env +++ b/.env @@ -3,3 +3,4 @@ VEGA_SIM_CONSOLE_TAG=develop VEGA_DEFAULT_KEY_NAME='Key 1' VEGA_SIM_NETWORKS_INTERNAL_TAG=main VEGA_SIM_NETWORKS_TAG=master +VEGA_SIM_CAPSULE_TAG=main diff --git a/Dockerfile b/Dockerfile index 58716a311..fdb2ef4c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,11 +5,14 @@ WORKDIR /extern RUN mkdir /extern/bin RUN ls -l RUN cd ./vega && CGO_ENABLED=0 go build -o ../bin/ ./... && cd .. +RUN cd ./vegacapsule && go build -o ../bin/ ./... && cd .. FROM python:3.10-slim-bullseye AS vegasim_base RUN useradd -ms /bin/bash vega +ENV PATH="/vega_market_sim/vega_sim/bin:${PATH}" + WORKDIR /vega_market_sim COPY ./requirements.txt . @@ -49,5 +52,8 @@ COPY ./pyproject.toml ./pyproject.toml RUN pip install -e . RUN chmod 777 /vega_market_sim +RUN chmod 777 /vega_market_sim/vega_sim/bin + +FROM docker:dind AS vegasim_dind + -USER vega diff --git a/Makefile b/Makefile index d6357ae91..59364a332 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ ui: pull_deps_ui build_deps_ui networks: pull_deps_networks build_deps_networks +capsule: pull_deps_capsule build_deps_capsule + proto: build_proto black clean_ui: @@ -70,6 +72,21 @@ build_deps_networks: @mkdir -p ./vega_sim/bin/networks @rsync -av ${EXTERN_DIR}/networks vega_sim/bin/ --exclude ${EXTERN_DIR}/networks/.git +pull_deps_capsule: + @if [ ! -d ./extern/ ]; then mkdir ./extern/; fi + @echo "Downloading Git dependencies into " ${EXTERN_DIR} + @echo "Downloading Vega vegacapsule" + @if [ ! -d ./extern/vegacapsule ]; then mkdir ./extern/vegacapsule; git clone https://github.com/vegaprotocol/vegacapsule ${EXTERN_DIR}/vegacapsule; fi +ifneq (${VEGA_SIM_CAPSULE_TAG},main) + @git -C ${EXTERN_DIR}/vegacapsule pull; git -C ${EXTERN_DIR}/vegacapsule checkout ${VEGA_SIM_CAPSULE_TAG} +else + @git -C ${EXTERN_DIR}/vegacapsule checkout main; git -C ${EXTERN_DIR}/vegacapsule pull +endif + +build_deps_capsule: + @mkdir -p ./vega_sim/bin + cd ${EXTERN_DIR}/vegacapsule && go build -o ../../vega_sim/bin/ ./... + build_proto: pull_deps @rm -rf ./vega_sim/proto @mkdir ./vega_sim/proto diff --git a/scripts/build-docker-learning.sh b/scripts/build-docker-learning.sh index 7009f7a7a..3ec575708 100755 --- a/scripts/build-docker-learning.sh +++ b/scripts/build-docker-learning.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64 --tag=vega_sim_learning:latest --target=vegasim_learning . \ No newline at end of file +docker buildx build --platform linux/amd64 --tag=vega_sim_learning:latest --target=vegasim_dind . \ No newline at end of file diff --git a/scripts/run-docker-capsule-test.sh b/scripts/run-docker-capsule-test.sh new file mode 100755 index 000000000..5d349c41f --- /dev/null +++ b/scripts/run-docker-capsule-test.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e + + +WORK_DIR="$(realpath "$(dirname "$0")/..")" +RESULT_DIR="${WORK_DIR}/test_logs/$(date '+%F_%H%M%S')-capsule" +mkdir -p "${RESULT_DIR}" + + +docker rm capsule_test || true +docker run \ + --platform linux/amd64 \ + --name capsule_test \ + -v /var/run/docker.sock:/var/run/docker.sock:rw \ + -p 4646:4646 \ + vega_sim_learning:latest \ + python -m vega_sim.scenario.fuzzed_markets.run_capsule_test --steps $1 \ + diff --git a/vega_sim/api/data.py b/vega_sim/api/data.py index 59ef434cd..a09d93159 100644 --- a/vega_sim/api/data.py +++ b/vega_sim/api/data.py @@ -763,6 +763,7 @@ def find_market_id( def find_asset_id( symbol: str, data_client: vac.VegaTradingDataClientV2, + enabled: bool = True, raise_on_missing: bool = False, ) -> str: """Looks up the Asset ID of a given asset name @@ -783,6 +784,8 @@ def find_asset_id( # Find settlement asset for asset in assets: if asset.details.symbol == symbol: + if (enabled) and (asset.status != vega_protos.assets.Asset.STATUS_ENABLED): + continue return asset.id if raise_on_missing: raise MissingAssetError( @@ -1613,3 +1616,19 @@ def estimate_position( ) return converted_margin_estimate, converted_liquidation_estimate + + +def get_stake( + data_client: vac.trading_data_grpc_v2, party_id: str, asset_decimals: int +): + return num_from_padded_int( + data_raw.get_stake(data_client=data_client, party_id=party_id), + decimals=asset_decimals, + ) + + +def get_asset( + data_client: vac.trading_data_grpc_v2, + asset_id: str, +): + return data_raw.asset_info(data_client=data_client, asset_id=asset_id) diff --git a/vega_sim/api/data_raw.py b/vega_sim/api/data_raw.py index 0f6db7295..55af703ee 100644 --- a/vega_sim/api/data_raw.py +++ b/vega_sim/api/data_raw.py @@ -660,3 +660,15 @@ def estimate_position( response = data_client.EstimatePosition(base_request) return response.margin, response.liquidation + + +def get_stake(data_client: vac.trading_data_grpc_v2, party_id: str = None): + base_request = data_node_protos_v2.trading_data.GetStakeRequest(party_id=party_id) + return data_client.GetStake(base_request).current_stake_available + + +def list_all_network_history_segments(data_client: vac.trading_data_grpc_v2): + base_request = ( + data_node_protos_v2.trading_data.ListAllNetworkHistorySegmentsRequest() + ) + return data_client.ListAllNetworkHistorySegments(base_request).segments diff --git a/vega_sim/api/market.py b/vega_sim/api/market.py index 6e99da722..a87618dbf 100644 --- a/vega_sim/api/market.py +++ b/vega_sim/api/market.py @@ -101,7 +101,19 @@ class MarketConfig(Config): "lp_price_range": 0.5, "linear_slippage_factor": 1e-3, "quadratic_slippage_factor": 0, - } + }, + "proposal": { + "decimal_places": 1, + "price_monitoring_parameters": "proposal", + "liquidity_monitoring_parameters": "proposal", + "log_normal": "proposal", + "position_decimal_places": 4, + "lp_price_range": 0.8, + "linear_slippage_factor": 0.001, + "quadratic_slippage_factor": 0.0, + "instrument": "default", + "metadata": None, + }, } def load(self, opt: Optional[str] = None): @@ -168,7 +180,39 @@ class PriceMonitoringParameters(Config): "probability": "0.90001", "auction_extension": 3600, }, - ] + ], + "proposal": [ + {"horizon": 60, "probability": "0.9999", "auction_extension": 120}, + {"horizon": 600, "probability": "0.9999", "auction_extension": 120}, + {"horizon": 3600, "probability": "0.9999", "auction_extension": 120}, + { + "horizon": 14400, + "probability": "0.9999", + "auction_extension": 180, + }, + { + "horizon": 43200, + "probability": "0.9999", + "auction_extension": 300, + }, + ], + # "proposal": [ + # { + # "horizon": 3600, + # "probability": "0.9999", + # "auction_extension": 120, + # }, + # { + # "horizon": 14400, + # "probability": "0.9999", + # "auction_extension": 180, + # }, + # { + # "horizon": 43200, + # "probability": "0.9999", + # "auction_extension": 300, + # }, + # ], } def load(self, opt: Optional[str] = None): @@ -185,7 +229,12 @@ class LiquidityMonitoringParameters(Config): "triggering_ratio": "0.7", "auction_extension": 0, "target_stake_parameters": "default", - } + }, + "proposal": { + "target_stake_parameters": "proposal", + "triggering_ratio": "0.7", + "auction_extension": 1, + }, } def load(self, opt: Optional[str] = None): @@ -211,7 +260,8 @@ class TargetStakeParameters(Config): "default": { "time_window": 60 * 60, "scaling_factor": 1, - } + }, + "proposal": {"time_window": 3600, "scaling_factor": 1}, } def load(self, opt: Optional[str] = None): @@ -233,7 +283,12 @@ class LogNormalRiskModel(Config): "risk_aversion_parameter": 0.01, "tau": 1.90128526884173e-06, "params": "default", - } + }, + "proposal": { + "risk_aversion_parameter": 0.000001, + "tau": 0.0001140771161, + "params": "proposal", + }, } def load(self, opt: Optional[str] = None): @@ -258,7 +313,8 @@ class LogNormalModelParams(Config): "mu": 0, "r": 0.016, "sigma": 3.0, - } + }, + "proposal": {"sigma": 1.5, "mu": 0, "r": 0}, } def __init__(self, opt: Optional[str] = None) -> None: diff --git a/vega_sim/api/trading.py b/vega_sim/api/trading.py index 863c9d162..c48c0c61d 100644 --- a/vega_sim/api/trading.py +++ b/vega_sim/api/trading.py @@ -240,7 +240,7 @@ def cancel_order( wallet_name: str, wallet: Wallet, market_id: str, - order_id: str, + order_id: Optional[str], key_name: Optional[str] = None, ): """ diff --git a/vega_sim/environment/agent.py b/vega_sim/environment/agent.py index fe0f70ebe..03b3b7225 100644 --- a/vega_sim/environment/agent.py +++ b/vega_sim/environment/agent.py @@ -1,8 +1,9 @@ from abc import ABC from dataclasses import dataclass -from typing import Any, Tuple, Dict, Optional, List +from typing import Any, Tuple, Dict, Optional, List, Union -from vega_sim.service import VegaService +from vega_sim.null_service import VegaServiceNull +from vega_sim.network_service import VegaServiceNetwork @dataclass @@ -17,11 +18,14 @@ class Agent(ABC): def __init__(self, tag: Optional[str] = None): self.tag = tag - def step(self, vega: VegaService): + def step(self, vega: Union[VegaServiceNull, VegaServiceNetwork]): pass def initialise( - self, vega: VegaService, create_key: bool = False, mint_key: bool = False + self, + vega: Union[VegaServiceNull, VegaServiceNetwork], + create_key: bool = False, + mint_key: bool = False, ): self.vega = vega @@ -74,7 +78,9 @@ def __init__( def step(self): pass - def initialise(self, vega: VegaService, create_key: bool = True): + def initialise( + self, vega: Union[VegaServiceNull, VegaServiceNetwork], create_key: bool = True + ): super().initialise(vega=vega) if create_key: self.vega.create_key( diff --git a/vega_sim/environment/environment.py b/vega_sim/environment/environment.py index e0eecf263..1c92af1d7 100644 --- a/vega_sim/environment/environment.py +++ b/vega_sim/environment/environment.py @@ -70,7 +70,7 @@ def __init__( step_length_seconds: Optional[int] = None, vega_service: Optional[VegaServiceNull] = None, pause_every_n_steps: Optional[int] = None, - random_state: np.random.RandomState = None, + random_state: Optional[np.random.RandomState] = None, ): """Set up a Vega protocol environment with some specified agents. Handles the entire Vega setup and environment lifetime process, allowing the @@ -167,13 +167,16 @@ def run( def _start_live_feeds(self, vega: VegaService): # Get lists of unique market_ids and party_ids to observe - market_ids = list( - { - vega.find_market_id(name=agent.market_name) + + market_ids = [ + vega.find_market_id(market_name) + for market_name in { + agent.market_name for agent in self.agents if hasattr(agent, "market_name") } - ) + ] + party_ids = list( { vega.wallet.public_key( @@ -425,6 +428,8 @@ def __init__( raise_datanode_errors: Optional[bool] = True, raise_step_errors: Optional[bool] = True, random_state: np.random.RandomState = None, + create_keys: Optional[bool] = False, + mint_keys: Optional[bool] = False, ): super().__init__( agents=agents, @@ -441,6 +446,9 @@ def __init__( self.raise_datanode_errors = raise_datanode_errors self.raise_step_errors = raise_step_errors + self.create_keys = create_keys + self.mint_keys = mint_keys + def run( self, run_with_console: bool = False, @@ -472,7 +480,9 @@ def _run( # Initialise agents without minting assets for agent in self.agents: - agent.initialise(vega=vega, create_key=False, mint_key=False) + agent.initialise( + vega=vega, create_key=self.create_keys, mint_key=self.mint_keys + ) self._start_live_feeds(vega=vega) @@ -498,8 +508,14 @@ def _run( logger.info(f"Completed {i} steps") for agent in self.agents: - agent.finalise() - + try: + agent.finalise() + except Exception as e: + msg = f"Agent '{agent.key_name}' failed to step. Error: {e}" + if self.raise_step_errors: + raise (e) + else: + logging.debug(msg) if pause_at_completion: input( "Environment run completed. Pausing to allow inspection of state." @@ -518,11 +534,11 @@ def step(self, vega: VegaServiceNetwork) -> None: try: agent.step(state) except Exception as e: - msg = f"Agent '{agent.key_name}' failed to step. Error: {e}" + msg = f"Agent '{agent.name()}' failed to step. Error: {e}" if self.raise_step_errors: raise e(msg) else: - logging.warning(msg) + logging.debug(msg) class RealtimeMarketEnvironment(MarketEnvironmentWithState): diff --git a/vega_sim/local_data_cache.py b/vega_sim/local_data_cache.py index 5495ad751..ef5602749 100644 --- a/vega_sim/local_data_cache.py +++ b/vega_sim/local_data_cache.py @@ -38,6 +38,7 @@ def _queue_forwarder( sink: Queue[Any], market_id: Optional[str] = None, party_id: Optional[str] = None, + kill_thread_sig: Optional[threading.Event()] = None, ) -> None: obs = data_raw.observe_event_bus( data_client=data_client, @@ -53,6 +54,8 @@ def _queue_forwarder( try: for o in obs: for event in o.events: + if kill_thread_sig.is_set(): + return output = handlers[event.type](event) if isinstance(output, (list, GeneratorType)): for elem in output: @@ -310,6 +313,7 @@ def start_live_feeds( if party_ids is not None else None ), + self._kill_thread_sig, ), daemon=True, ) diff --git a/vega_sim/network_service.py b/vega_sim/network_service.py index af9ca7955..51dc0f140 100644 --- a/vega_sim/network_service.py +++ b/vega_sim/network_service.py @@ -59,6 +59,8 @@ from vega_sim.constants import DATA_NODE_GRPC_PORT from vega_sim.scenario.constants import Network +from vega_sim.api.helpers import num_to_padded_int, num_from_padded_int + logger = logging.getLogger(__name__) @@ -218,7 +220,13 @@ def __init__( network_config_path: Optional[str] = None, wallet_path: Optional[str] = None, vega_home_path: Optional[str] = None, + wallet_home_path: Optional[str] = None, wallet_token_path: Optional[str] = None, + wallet_passphrase_path: Optional[str] = None, + faucet_url: Optional[bool] = None, + load_existing_keys: Optional[bool] = None, + governance_symbol: Optional[str] = "VEGA", + vegacapsule_bin_path: Optional[str] = "vegacapsule", ): """Method initialises the class. @@ -247,7 +255,11 @@ def __init__( """ # Run init method inherited from VegaService with network arguments. - super().__init__(can_control_time=False, warn_on_raw_data_access=False) + super().__init__( + can_control_time=False, + warn_on_raw_data_access=False, + governance_symbol=governance_symbol, + ) self.network = network self.run_with_wallet = run_with_wallet @@ -257,6 +269,9 @@ def __init__( self._wallet_url = None self._data_node_grpc_url = None self._data_node_query_url = None + self._data_node_rest_url = None + self._faucet_url = faucet_url + self._network_config = None self.vega_console_path = ( @@ -279,6 +294,12 @@ def __init__( self._vega_home = ( vega_home_path if vega_home_path is not None else environ.get("VEGA_HOME") ) + self._wallet_home = ( + wallet_home_path + if wallet_home_path is not None + else environ.get("WALLET_HOME") + ) + self._passphrase_file_path = wallet_passphrase_path self._token_path = ( wallet_token_path @@ -298,6 +319,18 @@ def __init__( raise ValueError( f"ERROR! {self.network.name.lower()} network config could not be found" ) + self._grpc_endpoints = itertools.cycle( + self.network_config["API"]["GRPC"]["Hosts"] + ) + self._rest_endpoints = itertools.cycle( + self.network_config["API"]["REST"]["Hosts"] + ) + self._graphql_endpoints = itertools.cycle( + self.network_config["API"]["GraphQL"]["Hosts"] + ) + + self.load_existing_keys = load_existing_keys + self.vegacapsule_bin_path = vegacapsule_bin_path self.log_dir = tempfile.mkdtemp(prefix="vega-sim-") @@ -394,21 +427,26 @@ def wait_for_total_catchup(self) -> None: def network_config(self) -> dict: if self._network_config is None: self._network_config = toml.load(self._network_config_path) - add_network_config(self._network_config_path) + if self.run_with_wallet: + add_network_config(self._network_config_path) return self._network_config @property def data_node_grpc_url(self) -> str: if self._data_node_grpc_url is None: - url = self.network_config["API"]["GRPC"]["Hosts"][0] - self._data_node_grpc_url = f"{url.split(':')[0]}:{DATA_NODE_GRPC_PORT}" + self._data_node_grpc_url = next(self._grpc_endpoints) return self._data_node_grpc_url + @property + def data_node_rest_url(self) -> str: + if self._data_node_rest_url is None: + self._data_node_rest_url = next(self._rest_endpoints) + return self._data_node_rest_url + @property def data_node_query_url(self) -> str: if self._data_node_query_url is None: - url = self.network_config["API"]["GraphQL"]["Hosts"][0] - self._data_node_query_url = url + self._data_node_query_url = next(self._graphql_endpoints) return self._data_node_query_url @property @@ -417,15 +455,27 @@ def wallet_url(self) -> str: self._wallet_url = f"http://127.0.0.1:1789" return self._wallet_url + @property + def faucet_url(self) -> str: + return self._faucet_url + @property def wallet(self) -> Wallet: if self._wallet is None: - self._wallet = VegaWallet.from_json( - self._token_path, - self.wallet_url, - wallet_path=self._wallet_path, - vega_home_dir=self._vega_home, - ) + if self.load_existing_keys: + self._wallet = VegaWallet.from_json( + self._token_path, + self.wallet_url, + wallet_path=self._wallet_path, + vega_home_dir=self._vega_home, + ) + else: + self._wallet = VegaWallet( + wallet_url=self.wallet_url, + wallet_path=self._wallet_path, + vega_home_dir=self._wallet_home, + passphrase_file_path=self._passphrase_file_path, + ) return self._wallet @property @@ -518,11 +568,11 @@ def switch_datanode(self, max_attempts: Optional[int] = -1): """ attempts = 0 - for url in itertools.cycle(self.network_config["API"]["GRPC"]["Hosts"]): + while True: attempts += 1 try: - self._data_node_grpc_url = f"{url.split(':')[0]}:{DATA_NODE_GRPC_PORT}" + self._data_node_grpc_url = next(self._grpc_endpoints) logging.debug(f"Switched to endpoint {self._data_node_grpc_url}") @@ -543,29 +593,29 @@ def switch_datanode(self, max_attempts: Optional[int] = -1): # Ping the datanode to check it is not behind self.ping_datanode() logging.debug( - f"Connection to endpoint {self._data_node_grpc_url} successful." + f"Connection to endpoint {self.data_node_grpc_url} successful." ) return except grpc._channel._InactiveRpcError: logging.warning( - f"Connection to endpoint {self._data_node_grpc_url} inactive." + f"Connection to endpoint {self.data_node_grpc_url} inactive." ) except grpc.FutureTimeoutError: logging.warning( - f"Connection to endpoint {self._data_node_grpc_url} timed out." + f"Connection to endpoint {self.data_node_grpc_url} timed out." ) except DatanodeBehindError: logging.warning( - f"Connection to endpoint {self._data_node_grpc_url} is behind." + f"Connection to endpoint {self.data_node_grpc_url} is behind." ) except DatanodeSlowResponseError: logging.warning( - f"Connection to endpoint {self._data_node_grpc_url} is slow." + f"Connection to endpoint {self.data_node_grpc_url} is slow." ) if attempts == max_attempts: @@ -573,6 +623,149 @@ def switch_datanode(self, max_attempts: Optional[int] = -1): raise Exception("Unable to establish connection to a data-node.") + def mint( + self, + key_name: Optional[str], + asset: str, + amount: float, + wallet_name: Optional[str] = None, + ) -> None: + """Mints a given amount of requested asset into the associated wallet + + Args: + wallet_name: + str, The name of the wallet + asset: + str, The ID of the asset to mint + amount: + float, the amount of asset to mint + key_name: + Optional[str], key name stored in metadata. Defaults to None. + """ + details = self.get_asset(asset).details + is_erc20 = True if details.erc20.contract_address != "" else False + + if is_erc20: + self.deposit( + symbol=details.symbol, + amount=amount, + key_name=key_name, + wallet_name=wallet_name, + ) + else: + max_faucet_amount = num_from_padded_int( + details.builtin_asset.max_faucet_amount_mint, details.decimals + ) + super().mint( + key_name=key_name, + asset=asset, + amount=amount if amount < max_faucet_amount else max_faucet_amount, + wallet_name=wallet_name, + ) + + def deposit( + self, + symbol: str, + amount: float, + key_name: str, + wallet_name: Optional[str] = None, + ): + """Deposit an amount of a specified ERC20 asset to a a vega key. + + Args: + symbol (str): Symbol of the asset to be deposited. + amount (float): Amount to be deposited. + key_name (str): Name of the key used for the deposit. + wallet_name (Optional[str], optional): Name of the wallet. Defaults to the + default wallet_name set in the wallet. + + Raises: + Exception: Raised if the deposit does not arrive within a fixed time limit. + """ + + asset_id = self.find_asset_id(symbol=symbol) + amount = str( + num_to_padded_int( + amount, + self.asset_decimals[asset_id], + ) + ) + + pub_key = self.wallet.public_key(name=key_name, wallet_name=wallet_name) + account_before = self.party_account(party_id=pub_key, asset_id=asset_id).general + + args = [ + self.vegacapsule_bin, + "ethereum", + "asset", + "deposit", + "--asset-symbol", + symbol, + "--amount", + amount, + "--pubkey", + pub_key, + ] + subprocess.run(args) + + for _ in range(500): + if self.party_account(party_id=pub_key, asset_id=asset_id) > account_before: + return + self.wait_fn(1) + raise Exception("Deposit never arrived.") + + def stake( + self, + amount: float, + key_name: str, + wallet_name: Optional[str] = None, + ): + """Stake a specified amount of a governance asset. + + Args: + amount (float): The amount to stake. + key_name (str): The name of the key used for staking. + wallet_name (Optional[str], optional): The name of the wallet. Defaults to + the default value set in the wallet. + + Raises: + Exception: Raised if the stake does not arrive within a given time limit. + """ + amount = str( + num_to_padded_int( + amount, + self.asset_decimals[self.find_asset_id(symbol=self.governance_symbol)], + ) + ) + + pub_key = self.wallet.public_key(name=key_name, wallet_name=wallet_name) + stake_before = self.get_stake(party_id=pub_key) + + args = [ + self.vegacapsule_bin_path, + "ethereum", + "asset", + "stake", + "--amount", + amount, + "--asset-symbol", + self.governance_symbol, + "--pub-key", + pub_key, + "--home-path", + self._vega_home, + ] + subprocess.run(args) + + for _ in range(500): + if self.get_stake(party_id=pub_key) > stake_before: + return + self.wait_fn(1) + raise Exception("Stake never arrived.") + + def wait_fn(self, wait_multiple: float = 1) -> None: + time.sleep(wait_multiple) + if __name__ == "__main__": """Module Example""" diff --git a/vega_sim/reinforcement/v2/agents/puppets.py b/vega_sim/reinforcement/v2/agents/puppets.py index ac83d6492..ab4868fd7 100644 --- a/vega_sim/reinforcement/v2/agents/puppets.py +++ b/vega_sim/reinforcement/v2/agents/puppets.py @@ -4,7 +4,9 @@ from typing import Optional from logging import getLogger +import vega_sim.proto.vega as vega_protos from vega_sim.scenario.common.agents import StateAgentWithWallet, VegaService, VegaState +from vega_sim.service import PeggedOrder logger = getLogger(__name__) @@ -15,6 +17,11 @@ class Side(Enum): BUY = 2 +class ForcedSide(Enum): + SELL = 0 + BUY = 1 + + @dataclass class Action: pass @@ -26,6 +33,12 @@ class MarketOrderAction(Action): volume: float +@dataclass +class AtTouchOrderAction(Action): + side: ForcedSide + volume: float + + @dataclass class NoAction(Action): pass @@ -33,6 +46,7 @@ class NoAction(Action): class AgentType(Enum): MARKET_ORDER = auto() + AT_TOUCH = auto() class Puppet(StateAgentWithWallet): @@ -90,5 +104,47 @@ def step(self, vega_state: VegaState): logger.exception(traceback.format_exc()) -AGENT_TYPE_TO_AGENT = {AgentType.MARKET_ORDER: MarketOrderPuppet} -AGENT_TYPE_TO_ACTION = {AgentType.MARKET_ORDER: MarketOrderAction} +class AtTouchPuppet(Puppet): + def initialise(self, vega: VegaService, create_wallet: bool = True): + super().initialise(vega, create_wallet) + self.market_id = self.vega.find_market_id(name=self.market_name) + + def step(self, vega_state: VegaState): + if self.action is not None and self.action is not NoAction: + try: + self.vega.cancel_order( + trading_key=self.key_name, market_id=self.market_id + ) + self.vega.submit_order( + trading_key=self.key_name, + market_id=self.market_id, + side=( + "SIDE_BUY" + if self.action.side == ForcedSide.BUY + else "SIDE_SELL" + ), + volume=self.action.volume, + time_in_force=vega_protos.vega.Order.TimeInForce.TIME_IN_FORCE_GTC, + order_type=vega_protos.vega.Order.Type.TYPE_LIMIT, + pegged_order=PeggedOrder( + reference=( + vega_protos.vega.PeggedReference.PEGGED_REFERENCE_BEST_BID + if self.action.side == ForcedSide.BUY + else vega_protos.vega.PeggedReference.PEGGED_REFERENCE_BEST_ASK + ), + offset=0, + ), + wait=False, + ) + except: + logger.exception(traceback.format_exc()) + + +AGENT_TYPE_TO_AGENT = { + AgentType.MARKET_ORDER: MarketOrderPuppet, + AgentType.AT_TOUCH: AtTouchPuppet, +} +AGENT_TYPE_TO_ACTION = { + AgentType.MARKET_ORDER: MarketOrderAction, + AgentType.AT_TOUCH: AtTouchOrderAction, +} diff --git a/vega_sim/reinforcement/v2/learning_environment.py b/vega_sim/reinforcement/v2/learning_environment.py index e12900650..4bcbc5724 100644 --- a/vega_sim/reinforcement/v2/learning_environment.py +++ b/vega_sim/reinforcement/v2/learning_environment.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, Type, Optional from dataclasses import dataclass from vega_sim.reinforcement.v2.agents.puppets import ( AGENT_TYPE_TO_AGENT, @@ -24,9 +24,9 @@ class StepResult: class Environment: def __init__( self, - agents: Dict[str, AgentType], - agent_to_reward: Dict[str, BaseRewarder], - agent_to_state: Dict[str, State], + agents: Dict[str, Type[AgentType]], + agent_to_reward: Dict[str, Type[BaseRewarder]], + agent_to_state: Dict[str, Type[State]], scenario: Scenario, reset_vega_every_n_runs: int = 100, funds_per_run: float = 10_000, @@ -63,9 +63,10 @@ def _extract_observation(self, agent_name: str) -> State: asset_id=self._asset_id, ) - def step(self, actions: Dict[str, Action]) -> Dict[str, StepResult]: + def step(self, actions: Dict[str, Optional[Action]]) -> Dict[str, StepResult]: for agent_name, action in actions.items(): - self._puppets[agent_name].set_next_action(action=action) + if action is not None: + self._puppets[agent_name].set_next_action(action=action) self._scenario.env.step(self._vega) self._vega.wait_fn(1) @@ -73,10 +74,13 @@ def step(self, actions: Dict[str, Action]) -> Dict[str, StepResult]: for agent_name, reward_gen in self._agent_to_reward.items(): step_res[agent_name] = StepResult( observation=self._extract_observation(agent_name), - reward=reward_gen.get_reward(self._vega), + reward=self.calculate_reward(reward_gen), ) return step_res + def calculate_reward(self, rewarder: Type[BaseRewarder]) -> float: + return rewarder.get_reward(vega=self._vega) + def _reset_vega(self) -> None: self._vega.stop() self._vega = VegaServiceNull( diff --git a/vega_sim/reinforcement/v2/rewards.py b/vega_sim/reinforcement/v2/rewards.py index c4bcae973..374f14fdb 100644 --- a/vega_sim/reinforcement/v2/rewards.py +++ b/vega_sim/reinforcement/v2/rewards.py @@ -7,6 +7,7 @@ class Reward(Enum): PNL = auto() + SQ_INVENTORY_PENALTY = auto() class BaseRewarder(ABC): @@ -45,4 +46,18 @@ def get_reward(self, vega: VegaService) -> float: return reward -REWARD_ENUM_TO_CLASS = {Reward.PNL: PnlRewarder} +class SquareInventoryPenalty(BaseRewarder): + def get_reward(self, vega: VegaService) -> float: + posn = vega.positions_by_market( + key_name=self.agent_key, + wallet_name=self.agent_wallet, + market_id=self.market_id, + ) + + return (-1 * posn.open_volume**2) if posn is not None else 0 + + +REWARD_ENUM_TO_CLASS = { + Reward.PNL: PnlRewarder, + Reward.SQ_INVENTORY_PENALTY: SquareInventoryPenalty, +} diff --git a/vega_sim/reinforcement/v2/stable_baselines/environment.py b/vega_sim/reinforcement/v2/stable_baselines/environment.py index 9fd5aac59..ca63e88c6 100644 --- a/vega_sim/reinforcement/v2/stable_baselines/environment.py +++ b/vega_sim/reinforcement/v2/stable_baselines/environment.py @@ -2,23 +2,44 @@ import gymnasium as gym from gymnasium import spaces +from typing import Type, Optional +from enum import Enum +from stable_baselines3.common.callbacks import BaseCallback from vega_sim.reinforcement.v2.agents.puppets import ( - MarketOrderAction, AgentType, + ForcedSide, Side, + AGENT_TYPE_TO_ACTION, ) from vega_sim.reinforcement.v2.learning_environment import Environment -from vega_sim.reinforcement.v2.rewards import Reward +from vega_sim.reinforcement.v2.rewards import Reward, REWARD_ENUM_TO_CLASS from vega_sim.reinforcement.v2.stable_baselines.states import ( price_state_with_fees_obs_space, + position_state_with_fees_obs_space, ) -from vega_sim.reinforcement.v2.states import PriceStateWithFees, State +from vega_sim.reinforcement.v2.states import PriceStateWithFees, State, PositionOnly from vega_sim.scenario.registry import CurveMarketMaker + logger = logging.getLogger(__name__) +class ActionType(Enum): + MARKET = "market" + AT_TOUCH_ONE_SIDE = "at_touch_one_side" + + +ACTION_TO_AGENT = { + ActionType.MARKET: AgentType.MARKET_ORDER, + ActionType.AT_TOUCH_ONE_SIDE: AgentType.AT_TOUCH, +} + + +class ActionLoggerCallback(BaseCallback): + pass + + class SingleAgentVegaEnv(gym.Env): """Custom Environment that follows gym interface.""" @@ -26,11 +47,13 @@ class SingleAgentVegaEnv(gym.Env): def __init__( self, - action_type: str = "market", - state_type: State = PriceStateWithFees, + action_type: ActionType = ActionType.MARKET, + state_type: Type[State] = PriceStateWithFees, + reward_type: Reward = Reward.PNL, num_levels_state: int = 5, trade_volume: float = 1, steps_per_trading_session: int = 1000, + terminal_reward_type: Optional[Reward] = None, ): super().__init__() self.num_levels_state = num_levels_state @@ -40,6 +63,7 @@ def __init__( self.steps_per_trading_session = steps_per_trading_session self.current_step = 0 self.learner_name = "learner_1" + self.terminal_reward_type = terminal_reward_type # Define action and observation space # They must be gym.spaces objects @@ -57,6 +81,7 @@ def __init__( initial_asset_mint=1e8, step_length_seconds=1, block_length_seconds=1, + # market_maker_assumed_market_kappa=0.8, buy_intensity=5, sell_intensity=5, market_name="ETH", @@ -67,45 +92,67 @@ def __init__( ) self.env = Environment( - agents={self.learner_name: AgentType.MARKET_ORDER}, - agent_to_reward={self.learner_name: Reward.PNL}, - agent_to_state={self.learner_name: PriceStateWithFees}, + agents={self.learner_name: ACTION_TO_AGENT[self.action_type]}, + agent_to_reward={self.learner_name: reward_type}, + agent_to_state={self.learner_name: state_type}, scenario=scenario, ) - def _get_action_space(self, action_type: str) -> spaces.Space: - if action_type == "market": + def _get_action_space(self, action_type: ActionType) -> spaces.Space: + if action_type == ActionType.MARKET: return spaces.Discrete(3) + elif action_type == ActionType.AT_TOUCH_ONE_SIDE: + return spaces.Discrete(2) else: raise Exception(f"Action type {action_type} is not implemented") - def _get_observation_space(self, state_type: State) -> spaces.Space: + def _get_observation_space(self, state_type: Type[State]) -> spaces.Space: if state_type == PriceStateWithFees: return price_state_with_fees_obs_space(num_levels=self.num_levels_state) + if state_type == PositionOnly: + return position_state_with_fees_obs_space() + + def _action_conversion(self, action): + if self.action_type == ActionType.MARKET: + return AGENT_TYPE_TO_ACTION[AgentType.MARKET_ORDER]( + side=Side(action), volume=self.trade_volume + ) + elif self.action_type == ActionType.AT_TOUCH_ONE_SIDE: + return AGENT_TYPE_TO_ACTION[AgentType.AT_TOUCH]( + side=ForcedSide(action), volume=self.trade_volume + ) + else: + raise Exception(f"Action type {self.action_type} is not implemented") def step(self, action): step_res = self.env.step( - { - self.learner_name: MarketOrderAction( - side=Side(action), volume=self.trade_volume - ) - } + {self.learner_name: self._action_conversion(action=action)} )[self.learner_name] self.current_step += 1 + is_terminal = self.current_step >= self.steps_per_trading_session + + if is_terminal and self.terminal_reward_type is not None: + terminal_reward = REWARD_ENUM_TO_CLASS[self.terminal_reward_type]( + agent_key=self.learner_name, + asset_id=self.env._asset_id, + market_id=self.env._market_id, + ) + reward = step_res.reward + self.env.calculate_reward(terminal_reward) + else: + reward = step_res.reward + return ( step_res.observation.to_array(), - step_res.reward, - self.current_step >= self.steps_per_trading_session, + reward, + is_terminal, False, {}, ) def reset(self): self.env.reset() - step_res = self.env.step( - {self.learner_name: MarketOrderAction(side=Side.NONE, volume=0)} - )[self.learner_name] + step_res = self.env.step({self.learner_name: None})[self.learner_name] self.current_step = 0 return step_res.observation.to_array(), {} diff --git a/vega_sim/reinforcement/v2/stable_baselines/run.py b/vega_sim/reinforcement/v2/stable_baselines/run.py index a62311ba1..b0eff724c 100644 --- a/vega_sim/reinforcement/v2/stable_baselines/run.py +++ b/vega_sim/reinforcement/v2/stable_baselines/run.py @@ -1,14 +1,21 @@ -from stable_baselines3 import PPO +from stable_baselines3 import PPO, DQN import vega_sim.reinforcement.v2.stable_baselines.environment as env +from vega_sim.reinforcement.v2.states import PriceStateWithFees, PositionOnly if __name__ == "__main__": - e = env.SingleAgentVegaEnv(steps_per_trading_session=200) + e = env.SingleAgentVegaEnv( + action_type=env.ActionType.AT_TOUCH_ONE_SIDE, + steps_per_trading_session=200, + reward_type=env.Reward.PNL, + terminal_reward_type=env.Reward.SQ_INVENTORY_PENALTY, + state_type=PriceStateWithFees, + ) model = PPO( "MlpPolicy", e, verbose=1, tensorboard_log="./ppo_tensorboard/", - n_steps=200, - batch_size=100, + n_steps=600, + batch_size=50, ).learn(total_timesteps=1_000_000) diff --git a/vega_sim/reinforcement/v2/stable_baselines/states.py b/vega_sim/reinforcement/v2/stable_baselines/states.py index a726f991a..b0cc664b1 100644 --- a/vega_sim/reinforcement/v2/stable_baselines/states.py +++ b/vega_sim/reinforcement/v2/stable_baselines/states.py @@ -20,3 +20,14 @@ def price_state_with_fees_obs_space( ), dtype=np.float64, ) + + +def position_state_with_fees_obs_space( + min_position: float = -1000, + max_position: float = 1000, +) -> gym.spaces.Box: + return gym.spaces.Box( + low=np.array([min_position]), + high=np.array([max_position]), + dtype=np.float64, + ) diff --git a/vega_sim/reinforcement/v2/states.py b/vega_sim/reinforcement/v2/states.py index dc330888d..edfa38f5b 100644 --- a/vega_sim/reinforcement/v2/states.py +++ b/vega_sim/reinforcement/v2/states.py @@ -144,3 +144,34 @@ def from_vega( + [0] * max(0, 5 - len(book_state.sells)), trading_fee=fee, ) + + +@dataclass(frozen=True) +class PositionOnly(State): + position: float + + def to_array(self) -> np.array: + return np.nan_to_num( + np.array( + [ + self.position, + ] + ) + ) + + @classmethod + def from_vega( + cls, + vega: VegaService, + for_key_name: str, + market_id: str, + asset_id: str, + for_wallet_name: Optional[str] = None, + ) -> "PositionOnly": + position = vega.positions_by_market( + for_key_name, market_id, wallet_name=for_wallet_name + ) + + return cls( + position=position.open_volume if position else 0, + ) diff --git a/vega_sim/scenario/common/agents.py b/vega_sim/scenario/common/agents.py index 4958b19c0..1c80805a8 100644 --- a/vega_sim/scenario/common/agents.py +++ b/vega_sim/scenario/common/agents.py @@ -930,9 +930,7 @@ def __init__( self.initial_mint = ( initial_mint if initial_mint is not None - else (2 * commitment_amount) - if commitment_amount is not None - else 100 + else (2 * commitment_amount) if commitment_amount is not None else 100 ) self.market_name = market_name @@ -1249,16 +1247,28 @@ def _scale_orders( buy_shape: List[MMOrder], sell_shape: List[MMOrder], ): + capped_buy_shape = [ + MMOrder(min([order.size, self.max_order_size]), order.price) + for order in buy_shape + ] + capped_sell_shape = [ + MMOrder( + min([order.size, self.max_order_size]), + order.price, + ) + for order in sell_shape + ] + buy_scaling_factor = ( self.safety_factor * self.commitment_amount * self.stake_to_ccy_volume ) / self._calculate_liquidity( - orders=buy_shape, + orders=capped_buy_shape, ) sell_scaling_factor = ( self.safety_factor * self.commitment_amount * self.stake_to_ccy_volume ) / self._calculate_liquidity( - orders=sell_shape, + orders=capped_sell_shape, ) # Scale the shapes @@ -2020,9 +2030,25 @@ def step(self, vega_state: VegaState): vega_state (VegaState): Object describing the state of the network and the market. """ - self.current_step += 1 + account = self.vega.party_account( + key_name=self.key_name, + wallet_name=self.wallet_name, + market_id=self.market_id, + asset_id=self.asset_id, + ) + + total_balance = account.general + account.margin + account.bond + + if total_balance == 0: + self.vega.mint( + key_name=self.key_name, + wallet_name=self.wallet_name, + amount=self.initial_asset_mint, + asset=self.asset_id, + ) + if self.random_state.rand() <= self.submit_bias: self._submit_order(vega_state=vega_state) @@ -2030,6 +2056,10 @@ def step(self, vega_state: VegaState): self._cancel_order(vega_state=vega_state) def _submit_order(self, vega_state: VegaState): + if self.current_step > len(self.price_process) - 1: + print("too long") + return + # Calculate reference_buy_price and reference_sell_price of price distribution if (self.spread is None) or (self.price_process is None): # If agent does not have price_process data, offset orders from best bid/ask @@ -2064,14 +2094,22 @@ def _submit_order(self, vega_state: VegaState): if side == "SIDE_BUY": volume = self.buy_volume * self.random_state.poisson(self.buy_intensity) - price = reference_buy_price + (random_offset - ln_mean) + # price = reference_buy_price + (random_offset - ln_mean) + price = reference_sell_price + (random_offset - ln_mean) elif side == "SIDE_SELL": volume = self.sell_volume * self.random_state.poisson(self.sell_intensity) - price = reference_sell_price - (random_offset - ln_mean) + price = reference_buy_price - (random_offset - ln_mean) + # price = reference_sell_price - (random_offset - ln_mean) expires_at = self.vega.get_blockchain_time() + self.duration * 1e9 + # bounds = self.vega.price_bounds(self.market_id) + # if bounds[0] and self.vega.best_prices(self.market_id)[0] < bounds[0]: + # import pdb + + # pdb.set_trace() + self.vega.submit_order( trading_wallet=self.wallet_name, market_id=self.market_id, @@ -2311,7 +2349,7 @@ def _close_positions(self, order: ITOrder): market_id=self.market_id, side=side, volume=order.volume, - wait=True, + wait=False, fill_or_kill=False, trading_key=self.key_name, ) diff --git a/vega_sim/scenario/configurable_market/agents.py b/vega_sim/scenario/configurable_market/agents.py index a35697347..2d6d4b375 100644 --- a/vega_sim/scenario/configurable_market/agents.py +++ b/vega_sim/scenario/configurable_market/agents.py @@ -38,6 +38,7 @@ def __init__( tag: Optional[str] = None, settlement_price: Optional[float] = None, initial_mint: Optional[float] = 1e9, + stake_key: bool = False, ): super().__init__( wallet_name=proposal_wallet_name, @@ -62,6 +63,8 @@ def __init__( self.settlement_price = settlement_price + self.stake_key = stake_key + def initialise( self, vega: Union[VegaServiceNull, VegaServiceNetwork], @@ -83,6 +86,12 @@ def initialise( amount=1e4, key_name=self.key_name, ) + if self.stake_key: + self.vega.stake( + amount=1, + key_name=self.key_name, + wallet_name=self.wallet_name, + ) self.vega.wait_for_total_catchup() diff --git a/vega_sim/scenario/configurable_market/scenario.py b/vega_sim/scenario/configurable_market/scenario.py index 6c8de4007..ea6e78ee6 100644 --- a/vega_sim/scenario/configurable_market/scenario.py +++ b/vega_sim/scenario/configurable_market/scenario.py @@ -39,10 +39,10 @@ class ConfigurableMarket(Scenario): def __init__( self, - market_name: str = None, - market_code: str = None, - asset_name: str = None, - asset_dp: str = None, + market_name: Optional[str] = None, + market_code: Optional[str] = None, + asset_name: Optional[str] = None, + asset_dp: Optional[str] = None, num_steps: int = 120, granularity: Optional[Granularity] = Granularity.MINUTE, block_size: int = 1, @@ -52,6 +52,8 @@ def __init__( ] = None, settle_at_end: bool = True, price_process_fn: Optional[Callable] = None, + market_config: Optional[MarketConfig] = None, + pause_every_n_steps: Optional[int] = None, ): super().__init__(state_extraction_fn=state_extraction_fn) @@ -62,6 +64,7 @@ def __init__( self.block_length_seconds = block_length_seconds self.settle_at_end = settle_at_end self.price_process_fn = price_process_fn + self.pause_every_n_steps = pause_every_n_steps # Asset parameters self.asset_name = asset_name @@ -70,12 +73,13 @@ def __init__( # Market parameters self.market_name = market_name self.market_code = market_code + self.market_config = market_config def _generate_price_process( self, random_state, ) -> list: - start = "2021-07-01 00:00:00" + start = "2022-10-01 00:00:00" start = datetime.strptime(start, "%Y-%m-%d %H:%M:%S") @@ -84,10 +88,11 @@ def _generate_price_process( end = start + timedelta(seconds=(self.num_steps + 1) * self.granularity.value) price_process = get_historic_price_series( - product_id="ETH-USD", + product_id="BTC-USD", granularity=self.granularity, start=str(start), end=str(end), + interpolation=f"{self.granularity.value}s", ) return list(price_process) @@ -96,10 +101,13 @@ def configure_agents( self, vega: VegaServiceNull, tag: str, - market_config: Optional[MarketConfig] = None, random_state: Optional[np.random.RandomState] = None, + **kwargs, ) -> Dict[str, StateAgent]: - market_config = market_config if market_config is not None else MarketConfig() + market_config = ( + self.market_config if self.market_config is not None else MarketConfig() + ) + market_config.load("proposal") random_state = ( random_state if random_state is not None else np.random.RandomState() @@ -132,13 +140,13 @@ def configure_agents( key_name=MAKER_PARTY.key_name, price_process_generator=iter(price_process), initial_asset_mint=1e9, - commitment_amount=1e6, + commitment_amount=10e6, market_name=market_name, asset_name=asset_name, market_decimal_places=market_config.decimal_places, asset_decimal_places=self.asset_decimal, num_steps=self.num_steps, - kappa=0.3, + kappa=0.6, num_levels=25, tick_spacing=1, inventory_upper_boundary=200, @@ -247,4 +255,5 @@ def configure_environment( transactions_per_block=self.block_size, vega_service=vega, block_length_seconds=self.block_length_seconds, + pause_every_n_steps=self.pause_every_n_steps, ) diff --git a/vega_sim/scenario/constants.py b/vega_sim/scenario/constants.py index 9497c24de..013e14af2 100644 --- a/vega_sim/scenario/constants.py +++ b/vega_sim/scenario/constants.py @@ -9,3 +9,4 @@ class Network(Enum): STAGNET3 = "vegawallet-stagnet3" FAIRGROUND = "vegawallet-fairground" TESTNET2 = "testnet2" + CAPSULE = "config" diff --git a/vega_sim/scenario/fuzzed_markets/agents.py b/vega_sim/scenario/fuzzed_markets/agents.py index 78b056757..98a499c3f 100644 --- a/vega_sim/scenario/fuzzed_markets/agents.py +++ b/vega_sim/scenario/fuzzed_markets/agents.py @@ -363,14 +363,15 @@ def step(self, vega_state): size = add_to_margin / (midprice * risk_factor + 1e-20) - self.vega.submit_market_order( - trading_key=self.key_name, - trading_wallet=self.wallet_name, - market_id=self.market_id, - side=self.side, - volume=size, - wait=False, - ) + if size > 0: + self.vega.submit_market_order( + trading_key=self.key_name, + trading_wallet=self.wallet_name, + market_id=self.market_id, + side=self.side, + volume=size, + wait=False, + ) class DegenerateLiquidityProvider(StateAgentWithWallet): diff --git a/vega_sim/scenario/fuzzed_markets/run_capsule_test.py b/vega_sim/scenario/fuzzed_markets/run_capsule_test.py new file mode 100644 index 000000000..c358f46a1 --- /dev/null +++ b/vega_sim/scenario/fuzzed_markets/run_capsule_test.py @@ -0,0 +1,156 @@ +import os +import time +import atexit +import logging +import argparse +import functools +import subprocess + +from io import BufferedWriter +from typing import Optional, List, Dict +from datetime import datetime + +from vega_sim.scenario.constants import Network +from vega_sim.network_service import VegaServiceNetwork +from vega_sim.scenario.fuzzed_markets.scenario import FuzzingScenario + + +def _terminate_proc( + proc: subprocess.Popen[bytes], out_file: BufferedWriter, err_file: BufferedWriter +) -> None: + subprocess.run( + [ + "vega_sim/bin/vegacapsule", + "network", + "destroy", + ] + ) + proc.terminate() + out_file.close() + err_file.close() + + +def _popen_process( + popen_args: List[str], + dir_root: str, + log_name: str, + env: Optional[Dict[str, str]] = None, +) -> subprocess.Popen[bytes]: + out = open(os.path.join(dir_root, f"{log_name}.out"), "wb") + err = open(os.path.join(dir_root, f"{log_name}.err"), "wb") + sub_proc = subprocess.Popen( + popen_args, stdout=out, stderr=err, env=env, close_fds=True + ) + + atexit.register(functools.partial(_terminate_proc, sub_proc, out, err)) + return sub_proc + + +def _run( + steps: int, + test_dir: Optional[str] = None, +): + test_dir = ( + test_dir + if test_dir is not None + else os.path.abspath( + f"test_logs/{datetime.now().strftime('%Y-%m-%d_%H%M%S')}-capsule" + ) + ) + if not os.path.exists(test_dir): + os.makedirs(test_dir) + + _popen_process( + [ + "vega_sim/bin/vegacapsule", + "nomad", + "--home-path", + f"{test_dir}/vegahome", + "--install-path", + "./vega_sim/bin", + ], + dir_root=test_dir, + log_name="nomad", + ) + # Sleep to allow nomad server to finish setup + time.sleep(10) + subprocess.run( + [ + "vega_sim/bin/vegacapsule", + "network", + "bootstrap", + "--config-path", + "vega_sim/vegacapsule/config.hcl", + "--home-path", + f"{test_dir}/vegahome", + ] + ) + subprocess.run( + [ + "vega_sim/bin/vegacapsule", + "ethereum", + "multisig", + "init", + "--home-path", + f"{test_dir}/vegahome", + ] + ) + + scenario = FuzzingScenario( + num_steps=steps, + step_length_seconds=2, + block_length_seconds=1, + transactions_per_block=4096, + n_markets=1, + ) + + with VegaServiceNetwork( + network=Network.CAPSULE, + run_with_wallet=False, + run_with_console=True, + vega_home_path=f"{test_dir}/vegahome", + wallet_home_path=f"{test_dir}/vegahome/wallet", + network_config_path=f"{test_dir}/vegahome/wallet", + wallet_passphrase_path="vega_sim/vegacapsule/passphrase-file", + faucet_url="http://localhost:1790", + load_existing_keys=False, + governance_symbol="VEGA", + ) as vega: + # Run the fuzzing scenario + scenario.run_iteration( + vega=vega, + network=Network.CAPSULE, + log_every_n_steps=100, + ) + + # Check all data nodes have matching network history segments + nodes_checked = set() + previous_node_segments = None + while vega.data_node_grpc_url not in nodes_checked: + # Check the network history segments of the current node match the previous + current_node_segments = vega.list_all_network_history_segments() + # print(current_node_segments) + if previous_node_segments is not None: + assert previous_node_segments == current_node_segments + previous_node_segments = current_node_segments + + # Add node to the checked nodes then switch node + nodes_checked.add(vega.data_node_grpc_url) + vega.switch_datanode() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("-s", "--steps", type=int, default=100) + parser.add_argument("--test-dir", type=str, default=None) + + parser.add_argument("-d", "--debug", action="store_true") + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) + + _run( + steps=args.steps, + test_dir=args.test_dir, + ) diff --git a/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py b/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py index a3fa5c0be..19c6f9f09 100644 --- a/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py +++ b/vega_sim/scenario/fuzzed_markets/run_fuzz_test.py @@ -21,10 +21,11 @@ def _run(steps: int = 2880, output: bool = False, output_dir: str = "fuzz_plots"): scenario = FuzzingScenario( num_steps=steps, - step_length_seconds=30, + step_length_seconds=60, block_length_seconds=1, transactions_per_block=4096, output=output, + n_markets=1, ) with VegaServiceNull( diff --git a/vega_sim/scenario/fuzzed_markets/scenario.py b/vega_sim/scenario/fuzzed_markets/scenario.py index 02ec93d08..3c5514c50 100644 --- a/vega_sim/scenario/fuzzed_markets/scenario.py +++ b/vega_sim/scenario/fuzzed_markets/scenario.py @@ -1,35 +1,40 @@ -from vega_sim.api.market import MarketConfig - import argparse +import datetime import logging +from dataclasses import dataclass +from datetime import timedelta +from typing import Dict, List, Optional + import numpy as np -from typing import Optional, List -from vega_sim.scenario.common.utils.price_process import random_walk +import pandas as pd -from vega_sim.scenario.scenario import Scenario -from vega_sim.environment.environment import MarketEnvironmentWithState +from vega_sim.api.market import MarketConfig +from vega_sim.environment.environment import ( + MarketEnvironmentWithState, + NetworkEnvironment, +) from vega_sim.null_service import VegaServiceNull - -from vega_sim.scenario.configurable_market.agents import ConfigurableMarketManager from vega_sim.scenario.common.agents import ( - StateAgent, - UncrossAuctionAgent, ExponentialShapedMarketMaker, - MarketOrderTrader, LimitOrderTrader, + MarketOrderTrader, + StateAgent, + UncrossAuctionAgent, ) +from vega_sim.scenario.common.utils.price_process import ( + Granularity, + get_historic_price_series, + random_walk, +) +from vega_sim.scenario.configurable_market.agents import ConfigurableMarketManager +from vega_sim.scenario.constants import Network from vega_sim.scenario.fuzzed_markets.agents import ( - FuzzingAgent, - DegenerateTrader, DegenerateLiquidityProvider, + DegenerateTrader, + FuzzingAgent, FuzzyLiquidityProvider, ) - -import datetime -from typing import Optional, Dict -from dataclasses import dataclass -from vega_sim.scenario.common.agents import ExponentialShapedMarketMaker -import pandas as pd +from vega_sim.scenario.scenario import Scenario @dataclass @@ -84,6 +89,31 @@ def additional_data_to_rows(data) -> List[pd.Series]: return results +def _create_historic_price_process( + random_state, + num_steps, + granularity: Granularity, + product_id: str, + start: Optional[datetime.datetime] = None, +) -> list: + if start is None: + start = "2022-10-01 00:00:00" + start = datetime.datetime.strptime(start, "%Y-%m-%d %H:%M:%S") + start = start + timedelta(days=int(random_state.choice(range(90)))) + + end = start + timedelta(seconds=(num_steps + 1) * granularity.value) + + price_process = get_historic_price_series( + product_id=product_id, + granularity=granularity, + start=str(start), + end=str(end), + interpolation=f"{granularity.value}s", + ) + + return list(price_process) + + def _create_price_process( random_state: np.random.RandomState, num_steps, decimal_places ): @@ -168,7 +198,7 @@ def configure_agents( random_state: Optional[np.random.RandomState], **kwargs, ) -> List[StateAgent]: - random_state = ( + self.random_state = ( random_state if random_state is not None else np.random.RandomState() ) @@ -183,6 +213,8 @@ def configure_agents( self.initial_asset_mint = 1e9 + start_dates = [datetime.datetime(2022, 11, 8), datetime.datetime(2023, 3, 12)] + for i_market in range(self.n_markets): # Define the market and the asset: market_name = f"ASSET_{str(i_market).zfill(3)}" @@ -191,25 +223,29 @@ def configure_agents( # Create fuzzed market config market_config = MarketConfig() - market_config.set( - "liquidity_monitoring_parameters.target_stake_parameters.scaling_factor", - 1e-4, - ) + market_config.load("proposal") + if self.fuzz_market_config is not None: - for param in self.fuzz_market_config: - market_config.set(param=self.fuzz_market_config[param]) + for param, value in self.fuzz_market_config.items(): + market_config.set(param, value) # Create fuzzed price process - price_process = _create_price_process( - random_state=random_state, - num_steps=self.num_steps, - decimal_places=int(market_config.decimal_places), + # price_process = _create_price_process( + # random_state=self.random_state, + # num_steps=self.num_steps, + # decimal_places=int(market_config.decimal_places), + # ) + price_process = _create_historic_price_process( + product_id="ETH-USD", + random_state=self.random_state, + num_steps=self.num_steps + 100, + granularity=Granularity.MINUTE, + start=start_dates[i_market], ) # Create fuzzed market managers market_managers.append( ConfigurableMarketManager( - proposal_wallet_name="MARKET_MANAGER", proposal_key_name="PROPOSAL_KEY", termination_wallet_name="MARKET_MANAGER", termination_key_name="TERMINATION_KEY", @@ -219,6 +255,7 @@ def configure_agents( asset_dp=asset_dp, asset_name=asset_name, settlement_price=price_process[-1], + stake_key=True if kwargs["network"] == Network.CAPSULE else False, tag=f"MARKET_{str(i_market).zfill(3)}", ) ) @@ -231,13 +268,15 @@ def configure_agents( initial_asset_mint=self.initial_asset_mint, market_name=market_name, asset_name=asset_name, - commitment_amount=1e6, + commitment_amount=1_000_000, market_decimal_places=market_config.decimal_places, asset_decimal_places=asset_name, num_steps=self.num_steps, - kappa=2.4, + kappa=0.3, tick_spacing=0.05, - market_kappa=50, + inventory_upper_boundary=2000, + inventory_lower_boundary=-2000, + market_kappa=0.3, state_update_freq=10, tag=f"MARKET_{str(i_market).zfill(3)}", ) @@ -262,12 +301,13 @@ def configure_agents( random_traders.append( MarketOrderTrader( wallet_name=f"RANDOM_TRADERS", + initial_asset_mint=100_000, key_name=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}", market_name=market_name, asset_name=asset_name, - buy_intensity=10, - sell_intensity=10, - base_order_size=1, + buy_intensity=1, + sell_intensity=1, + base_order_size=0.1, step_bias=1, tag=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}", ) @@ -280,17 +320,18 @@ def configure_agents( key_name=f"LIMIT_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}", market_name=market_name, asset_name=asset_name, - time_in_force_opts={"TIME_IN_FORCE_GTT": 1}, - buy_volume=1, - sell_volume=1, - buy_intensity=10, - sell_intensity=10, + # time_in_force_opts={"TIME_IN_FORCE_GTT": 1}, + buy_volume=0.1, + sell_volume=0.1, + buy_intensity=1, + sell_intensity=1, submit_bias=1, cancel_bias=0, duration=120, price_process=price_process, spread=0, - mean=-3, + initial_asset_mint=100_000_000, + mean=0, sigma=0.5, tag=f"MARKET_{str(i_market).zfill(3)}_AGENT_{str(i_agent).zfill(3)}", ) @@ -317,8 +358,8 @@ def configure_agents( market_name=market_name, asset_name=asset_name, side=side, - initial_asset_mint=1_000, - size_factor=0.7, + initial_asset_mint=10_000, + size_factor=0.35, step_bias=0.1, tag=f"MARKET_{str(i_market).zfill(3)}_SIDE_{side}_AGENT_{str(i_agent).zfill(3)}", ) @@ -369,10 +410,10 @@ def configure_agents( + market_makers + auction_traders + random_traders - + fuzz_traders + # + fuzz_traders + degenerate_traders + degenerate_liquidity_providers - + fuzz_liquidity_providers + # + fuzz_liquidity_providers ) return {agent.name(): agent for agent in agents} @@ -381,15 +422,28 @@ def configure_environment( vega: VegaServiceNull, **kwargs, ) -> MarketEnvironmentWithState: - return MarketEnvironmentWithState( - agents=list(self.agents.values()), - n_steps=self.num_steps, - random_agent_ordering=False, - transactions_per_block=self.transactions_per_block, - vega_service=vega, - step_length_seconds=self.step_length_seconds, - block_length_seconds=vega.seconds_per_block, - ) + if kwargs.get("network", Network.NULLCHAIN) == Network.NULLCHAIN: + return MarketEnvironmentWithState( + agents=list(self.agents.values()), + n_steps=self.num_steps, + random_agent_ordering=False, + transactions_per_block=self.transactions_per_block, + vega_service=vega, + step_length_seconds=self.step_length_seconds, + block_length_seconds=vega.seconds_per_block, + ) + else: + return NetworkEnvironment( + agents=list(self.agents.values()), + n_steps=self.num_steps, + vega_service=vega, + step_length_seconds=self.step_length_seconds, + raise_datanode_errors=kwargs.get("raise_datanode_errors", False), + raise_step_errors=kwargs.get("raise_step_errors", False), + random_state=self.random_state, + create_keys=True, + mint_keys=True, + ) if __name__ == "__main__": diff --git a/vega_sim/scenario/registry.py b/vega_sim/scenario/registry.py index 1243c0784..a66be8442 100644 --- a/vega_sim/scenario/registry.py +++ b/vega_sim/scenario/registry.py @@ -19,6 +19,58 @@ Granularity, ) +mc = MarketConfig() + +CONF = { + "decimal_places": "1", + "price_monitoring_parameters": { + "triggers": [ + { + "horizon": "60", + "probability": "0.9999", + "auction_extension": "5", + }, + { + "horizon": "600", + "probability": "0.9999", + "auction_extension": "30", + }, + { + "horizon": "3600", + "probability": "0.9999", + "auction_extension": "120", + }, + { + "horizon": "14400", + "probability": "0.9999", + "auction_extension": "180", + }, + { + "horizon": "43200", + "probability": "0.9999", + "auction_extension": "300", + }, + ] + }, + "liquidity_monitoring_parameters": { + "target_stake_parameters": {"time_window": "3600", "scaling_factor": 1}, + "triggering_ratio": "0.7", + "auction_extension": "1", + }, + "log_normal": { + "risk_aversion_parameter": 0.000001, + "tau": 0.0001140771161, + "params": {"sigma": 1.5}, + }, + "position_decimal_places": "4", + "lp_price_range": "0.8", + "linear_slippage_factor": "0.001", + "quadratic_slippage_factor": "0.0", +} +for c, v in CONF.items(): + mc.set(c, v) + + SCENARIOS = { "comprehensive_market": lambda: ComprehensiveMarket( market_name="ETH", @@ -168,11 +220,12 @@ market_order_trader_base_order_size=0.01, ), "configurable_market": lambda: ConfigurableMarket( - market_name="RESEARCH: Ethereum:USD Q3 (Daily)", - market_code="ETH:USD", + market_name="RESEARCH: Bitcoin:USD Q3 (Daily)", + market_code="BTC:USD", asset_name="tUSD", asset_dp=18, - num_steps=60 * 24, + num_steps=60 * 24 * 7, + market_config=None, ), "vega_load_test": lambda: VegaLoadTest( num_steps=60 * 24, diff --git a/vega_sim/service.py b/vega_sim/service.py index 6d4c66cfa..c4fbc9a1c 100644 --- a/vega_sim/service.py +++ b/vega_sim/service.py @@ -104,6 +104,7 @@ def __init__( warn_on_raw_data_access: bool = True, seconds_per_block: int = 1, listen_for_high_volume_stream_updates: bool = False, + governance_symbol: Optional[str] = "VOTE", ): """A generic service for accessing a set of Vega processes. @@ -138,6 +139,10 @@ def __init__( updates per block, such as all ledger transactions. For a network running at ~1s/block these are likely to be fine, but can be a hindrance working at full nullchain speed. + governance_symbol: + str, default "VOTE", allows the symbol of the governance asset to be + defined. This defaults to "VOTE" for nullchain networks but should + be changed (most likely to "VEGA") for other networks. """ self._core_client = None @@ -156,6 +161,8 @@ def __init__( ) self.seconds_per_block = seconds_per_block + self.governance_symbol = governance_symbol + @property def market_price_decimals(self) -> int: if self._market_price_decimals is None: @@ -454,23 +461,22 @@ def create_asset( name=name, symbol=symbol, decimals=decimals, - max_faucet_amount=max_faucet_amount * 10**decimals, + max_faucet_amount=num_to_padded_int(max_faucet_amount, decimals), quantum=quantum, data_client=self.trading_data_client_v2, - closing_time=blockchain_time_seconds + self.seconds_per_block * 90, - enactment_time=blockchain_time_seconds + self.seconds_per_block * 100, - validation_time=blockchain_time_seconds + self.seconds_per_block * 30, + closing_time=blockchain_time_seconds + self.seconds_per_block * 10, + enactment_time=blockchain_time_seconds + self.seconds_per_block * 10, + validation_time=blockchain_time_seconds + self.seconds_per_block * 5, time_forward_fn=lambda: self.wait_fn(2), key_name=key_name, ) - self.wait_fn(1) gov.approve_proposal( proposal_id=proposal_id, wallet_name=wallet_name, wallet=self.wallet, key_name=key_name, ) - self.wait_fn(110) + self.wait_fn(15) def create_market_from_config( self, @@ -486,8 +492,8 @@ def create_market_from_config( proposal_wallet_name=proposal_wallet_name, proposal_key_name=proposal_key_name, market_config=market_config, - closing_time=blockchain_time_seconds + self.seconds_per_block * 90, - enactment_time=blockchain_time_seconds + self.seconds_per_block * 90, + closing_time=blockchain_time_seconds + self.seconds_per_block * 10, + enactment_time=blockchain_time_seconds + self.seconds_per_block * 10, time_forward_fn=lambda: self.wait_fn(2), ) @@ -497,7 +503,7 @@ def create_market_from_config( wallet_name=proposal_wallet_name, key_name=proposal_key_name, ) - self.wait_fn(110) + self.wait_fn(15) def create_simple_market( self, @@ -568,8 +574,8 @@ def create_simple_market( ), position_decimals=position_decimals, market_decimals=market_decimals, - closing_time=blockchain_time_seconds + self.seconds_per_block * 90, - enactment_time=blockchain_time_seconds + self.seconds_per_block * 100, + closing_time=blockchain_time_seconds + self.seconds_per_block * 10, + enactment_time=blockchain_time_seconds + self.seconds_per_block * 10, risk_model=risk_model, time_forward_fn=lambda: self.wait_fn(2), price_monitoring_parameters=price_monitoring_parameters, @@ -581,7 +587,7 @@ def create_simple_market( wallet_name=wallet_name, key_name=proposal_key, ) - self.wait_fn(110) + self.wait_fn(15) def submit_market_order( self, @@ -846,7 +852,7 @@ def cancel_order( self, trading_key: str, market_id: str, - order_id: str, + order_id: Optional[str] = None, wallet_name: Optional[str] = None, ): """ @@ -1232,7 +1238,7 @@ def market_account( def best_prices( self, market_id: str, - ) -> Tuple[int, int]: + ) -> Tuple[float, float]: """ Output the best static bid price and best static ask price in current market. """ @@ -1469,12 +1475,16 @@ def find_market_id(self, name: str, raise_on_missing: bool = False) -> str: data_client=self.trading_data_client_v2, ) - def find_asset_id(self, symbol: str, raise_on_missing: bool = False) -> str: + def find_asset_id( + self, symbol: str, enabled: bool = True, raise_on_missing: bool = False + ) -> str: """Looks up the Asset ID of a given asset name Args: symbol: str, The symbol of the asset to look up + enabled: + bool, whether the asset must be enabled for the id to be returned raise_on_missing: bool, whether to raise an Error or silently return if the asset does not exist @@ -1485,6 +1495,7 @@ def find_asset_id(self, symbol: str, raise_on_missing: bool = False) -> str: return data.find_asset_id( symbol=symbol, raise_on_missing=raise_on_missing, + enabled=enabled, data_client=self.trading_data_client_v2, ) @@ -2400,3 +2411,24 @@ def estimate_position( else None, asset_decimals=self.asset_decimals, ) + + def get_stake( + self, + party_id: Optional[str] = None, + ): + asset_id = self.find_asset_id(symbol=self.governance_symbol) + return data.get_stake( + data_client=self.trading_data_client_v2, + party_id=party_id, + asset_decimals=self.asset_decimals[asset_id], + ) + + def get_asset(self, asset_id: str): + return data.get_asset( + data_client=self.trading_data_client_v2, asset_id=asset_id + ) + + def list_all_network_history_segments(self): + return data_raw.list_all_network_history_segments( + data_client=self.trading_data_client_v2 + ) diff --git a/vega_sim/tools/scenario_plots.py b/vega_sim/tools/scenario_plots.py index b56d619d2..891a715ce 100644 --- a/vega_sim/tools/scenario_plots.py +++ b/vega_sim/tools/scenario_plots.py @@ -364,20 +364,25 @@ def plot_price_comparison( ax0.set_ylim(ax0.get_ylim()) ax0.plot(external_price_series, linewidth=0.8, alpha=0.8) - ep_volatility = external_price_series.var() / external_price_series.size + ep_volatility = external_price_series.var() / ( + external_price_series.size + 0.000001 + ) mp_volatility = mark_price_series.var() / mark_price_series.size ax0.text( x=0.1, y=0.1, - s=f"external-price volatility = {round(ep_volatility, 1)}\nmark-price volatility = {round(mp_volatility, 1)}", + s=( + f"external-price volatility = {round(ep_volatility, 1)}\nmark-price" + f" volatility = {round(mp_volatility, 1)}" + ), fontsize=8, bbox=dict(facecolor="white", alpha=1), transform=ax0.transAxes, ) ax0.set_ylabel("PRICE") - ax0.legend(labels=["external price", "mark price"]) + ax0.legend(labels=["mark price", "external price"]) def plot_degen_close_outs( diff --git a/vega_sim/vegacapsule/config.hcl b/vega_sim/vegacapsule/config.hcl new file mode 100644 index 000000000..32ee6d193 --- /dev/null +++ b/vega_sim/vegacapsule/config.hcl @@ -0,0 +1,132 @@ +vega_binary_path = "./vega_sim/bin/vega" +vega_capsule_binary_path = "./vega_sim/bin/vegacapsule" + +network "testnet" { + ethereum { + chain_id = "1440" + network_id = "1441" + endpoint = "ws://127.0.0.1:8545/" + } + + faucet "faucet-1" { + wallet_pass = "f4uc3tw4ll3t-v3g4-p4ssphr4e3" + + template = <<-EOT +[RateLimit] + CoolDown = "1ns" + AllowList = ["10.0.0.0/8", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fe80::/10"] +[Node] + Port = 3002 + IP = "127.0.0.1" +EOT + } + + wallet "wallet-1" { + token_passphrase_path = "passphrase-file" + template = <<-EOT +Name = "capsule" +Level = "info" +TokenExpiry = "168h0m0s" +Port = 1789 +Host = "0.0.0.0" + +[API] + [API.GRPC] + Hosts = [ + "localhost:3027", + "localhost:3037", + ] + Retries = 5 + [API.REST] + Hosts = [ + "http://localhost:3028", + "http://localhost:3038", + ] + [API.GraphQL] + Hosts = [ + "http://localhost:3028/graphql", + "http://localhost:3038/graphql", + ] +EOT + } + + pre_start { + docker_service "ganache-1" { + image = "vegaprotocol/ganache:latest" + cmd = "ganache-cli" + args = [ + "--blockTime", "1", + "--chainId", "1440", + "--networkId", "1441", + "-h", "0.0.0.0", + "-p", "8545", + "-m", "ozone access unlock valid olympic save include omit supply green clown session", + "--db", "/app/ganache-db", + ] + static_port { + value = 8545 + to = 8545 + } + auth_soft_fail = true + } + docker_service "postgres-1" { + image = "vegaprotocol/timescaledb:2.8.0-pg14" + cmd = "postgres" + args = [] + env = { + POSTGRES_USER="vega" + POSTGRES_PASSWORD="vega" + POSTGRES_DBS="vega0,vega1,vega2,vega3,vega4,vega5,vega6,vega7,vega8,vega9,vega10,vega11,vega12,vega13,vega14,vega15,vega16,vega17,vega18,vega19,vega20,vega21,vega22,vega23,vega24,vega25" + } + + static_port { + value = 5232 + to = 5432 + } + resources { + cpu = 600 + memory = 900 + } + + volume_mounts = ["${network_home_path}:${network_home_path}"] + + auth_soft_fail = true + } + } + + genesis_template_file = "./genesis.tmpl" + smart_contracts_addresses_file = "./smart_contracts_addresses.json" + + node_set "validators" { + count = 2 + mode = "validator" + + node_wallet_pass = "n0d3w4ll3t-p4ssphr4e3" + vega_wallet_pass = "w4ll3t-p4ssphr4e3" + ethereum_wallet_pass = "ch41nw4ll3t-3th3r3um-p4ssphr4e3" + + config_templates { + vega_file = "./templates/vega_validators.tmpl" + tendermint_file = "./templates/tendermint_validators.tmpl" + } + } + + node_set "full" { + count = 2 + mode = "full" + use_data_node = true + + pre_start_probe { + postgres { + connection = "user=vega dbname=vega{{ .NodeNumber }} password=vega port=5232 sslmode=disable" + query = "select 10 + 10" + } + } + + config_templates { + vega_file = "./templates/vega_full.tmpl" + tendermint_file = "./templates/tendermint_full.tmpl" + data_node_file = "./templates/data_node_full.tmpl" + } + } +} diff --git a/vega_sim/vegacapsule/genesis.tmpl b/vega_sim/vegacapsule/genesis.tmpl new file mode 100644 index 000000000..3b227d8d4 --- /dev/null +++ b/vega_sim/vegacapsule/genesis.tmpl @@ -0,0 +1,78 @@ +{ + "app_state": { + "assets": { + "{{.GetVegaContractID "VEGA"}}": { + "min_lp_stake": "1", + "decimals": 18, + "name": "Vega", + "symbol": "VEGA", + "total_supply": "64999723000000000000000000", + "source": { + "erc20": { + "contract_address": "{{.GetEthContractAddr "VEGA"}}" + } + } + } + }, + "network": { + "ReplayAttackThreshold": 30 + }, + "network_parameters": { + "blockchains.ethereumConfig": "{\"network_id\": \"{{ .NetworkID }}\", \"chain_id\": \"{{ .ChainID }}\", \"collateral_bridge_contract\": { \"address\": \"{{.GetEthContractAddr "erc20_bridge_1"}}\" }, \"confirmations\": 3, \"staking_bridge_contract\": { \"address\": \"{{.GetEthContractAddr "staking_bridge"}}\", \"deployment_block_height\": 0}, \"token_vesting_contract\": { \"address\": \"{{.GetEthContractAddr "erc20_vesting"}}\", \"deployment_block_height\": 0 }, \"multisig_control_contract\": { \"address\": \"{{.GetEthContractAddr "MultisigControl"}}\", \"deployment_block_height\": 0 }}", + "governance.proposal.asset.minProposerBalance": "1", + "governance.proposal.asset.minVoterBalance": "1", + "governance.proposal.asset.minClose": "2s", + "governance.proposal.asset.minEnact": "2s", + "governance.proposal.asset.requiredParticipation": "0.00000000000000000000000015", + "governance.proposal.market.minProposerBalance": "1", + "governance.proposal.market.minVoterBalance": "1", + "governance.proposal.market.minClose": "2s", + "governance.proposal.market.minEnact": "2s", + "governance.proposal.market.requiredParticipation": "0.00000000000000000000000015", + "governance.proposal.updateMarket.minProposerBalance": "1", + "governance.proposal.updateMarket.minVoterBalance": "1", + "governance.proposal.updateMarket.minClose": "2s", + "governance.proposal.updateMarket.minEnact": "2s", + "governance.proposal.updateMarket.requiredParticipation": "0.00000000000000000000000015", + "governance.proposal.updateNetParam.minProposerBalance": "1", + "governance.proposal.updateNetParam.minVoterBalance": "1", + "governance.proposal.updateNetParam.minClose": "2s", + "governance.proposal.updateNetParam.minEnact": "2s", + "governance.proposal.updateNetParam.requiredParticipation": "0.00000000000000000000000015", + "market.auction.minimumDuration": "3s", + "market.fee.factors.infrastructureFee": "0.001", + "market.fee.factors.makerFee": "0.004", + "market.liquidity.stakeToCcyVolume": "0.3", + "market.liquidity.targetstake.triggering.ratio": "0.7", + "network.checkpoint.timeElapsedBetweenCheckpoints": "10s", + "reward.asset": "{{.GetVegaContractID "VEGA"}}", + "reward.staking.delegation.competitionLevel": "3.1", + "reward.staking.delegation.delegatorShare": "0.883", + "reward.staking.delegation.maxPayoutPerParticipant": "700000000000000000000", + "reward.staking.delegation.minimumValidatorStake": "3000000000000000000000", + "reward.staking.delegation.payoutDelay": "5m", + "reward.staking.delegation.payoutFraction": "0.007", + "spam.protection.delegation.min.tokens": "1000000000000000000", + "spam.protection.max.delegations": "390", + "spam.protection.max.proposals": "100", + "spam.protection.max.votes": "100", + "spam.protection.proposal.min.tokens": "1", + "spam.protection.voting.min.tokens": "1", + "validators.delegation.minAmount": "100000000000000000", + "validators.epoch.length": "5s", + "snapshot.interval.length": "100", + "validators.vote.required": "0.67" + }, + "network_limits": { + "propose_asset_enabled": true, + "propose_asset_enabled_from": "2021-09-01T00:00:00Z", + "propose_market_enabled": true, + "propose_market_enabled_from": "2021-09-01T00:00:00Z" + } + }, + "consensus_params": { + "block": { + "time_iota_ms": "1" + } + } +} \ No newline at end of file diff --git a/vega_sim/vegacapsule/passphrase-file b/vega_sim/vegacapsule/passphrase-file new file mode 100644 index 000000000..fc80254b6 --- /dev/null +++ b/vega_sim/vegacapsule/passphrase-file @@ -0,0 +1 @@ +pass \ No newline at end of file diff --git a/vega_sim/vegacapsule/smart_contracts_addresses.json b/vega_sim/vegacapsule/smart_contracts_addresses.json new file mode 100644 index 000000000..0e09837ae --- /dev/null +++ b/vega_sim/vegacapsule/smart_contracts_addresses.json @@ -0,0 +1,48 @@ +{ + "addr0": { + "priv": "a37f4c2a678aefb5037bf415a826df1540b330b7e471aa54184877ba901b9ef0", + "pub": "0xEe7D375bcB50C26d52E1A4a472D8822A2A22d94F" + }, + "MultisigControl": { + "Ethereum": "0xdEcdA30fd3449718304eA201A8f220eBdE25dd1E" + }, + "ERC20_Asset_Pool": { + "Ethereum": "0xAa1eDb6C25e6B5ff2c8EdAf68757Ae557178E6eE" + }, + "erc20_bridge_1": { + "Ethereum": "0x9708FF7510D4A7B9541e1699d15b53Ecb1AFDc54" + }, + "erc20_bridge_2": { + "Ethereum": "0x29e1eA1cfb78f7c34802C90198Cc24aDcBBE4AD0" + }, + "tBTC": { + "Ethereum": "0xb63D135B0a6854EEb765d69ca36210cC70BECAE0", + "Vega": "0x5cfa87844724df6069b94e4c8a6f03af21907d7bc251593d08e4251043ee9f7c" + }, + "tDAI": { + "Ethereum": "0x879B84eCA313D62CE4e5ED717939B42cBa9e53cb", + "Vega": "0x6d9d35f657589e40ddfb448b7ad4a7463b66efb307527fedd2aa7df1bbd5ea61" + }, + "tEURO": { + "Ethereum": "0x7ccE194dAEf2A4e5C23C78C9330D4c907eCA6980", + "Vega": "0x8b52d4a3a4b0ffe733cddbc2b67be273816cfeb6ca4c8b339bac03ffba08e4e4" + }, + "tUSDC": { + "Ethereum": "0x1b8a1B6CBE5c93609b46D1829Cc7f3Cb8eeE23a0", + "Vega": "0x993ed98f4f770d91a796faab1738551193ba45c62341d20597df70fea6704ede" + }, + "VEGA": { + "Ethereum": "0x67175Da1D5e966e40D11c4B2519392B2058373de", + "Vega": "0xb4f2726571fbe8e33b442dc92ed2d7f0d810e21835b7371a7915a365f07ccd9b" + }, + "VEGAv1": { + "Ethereum": "0x8fa21D653C1bF17741055f00dD55663Bc52a8362", + "Vega": "0xc1607f28ec1d0a0b36842c8327101b18de2c5f172585870912f5959145a9176c" + }, + "erc20_vesting": { + "Ethereum": "0xF41bD86d462D36b997C0bbb4D97a0a3382f205B7" + }, + "staking_bridge": { + "Ethereum": "0x9135f5afd6F055e731bca2348429482eE614CFfA" + } + } \ No newline at end of file diff --git a/vega_sim/vegacapsule/templates/data_node_full.tmpl b/vega_sim/vegacapsule/templates/data_node_full.tmpl new file mode 100644 index 000000000..c1e5e396c --- /dev/null +++ b/vega_sim/vegacapsule/templates/data_node_full.tmpl @@ -0,0 +1,61 @@ +GatewayEnabled = true +[SQLStore] + Enabled = true + [SQLStore.ConnectionConfig] + Database = "vega{{.NodeNumber}}" + Host = "localhost" + Password = "vega" + Port = 5232 + UseTransactions = true + Username = "vega" + +[API] + Level = "Info" + Port = 3{{ printf "%02d" .NodeNumber }}7 + CoreNodeGRPCPort = 3{{ printf "%02d" .NodeNumber }}2 + [API.RateLimit] + Enabled = false + +[Pprof] + Level = "Info" + Enabled = true + Port = {{ printf "%02d" (add .NodeNumber 60) }}60 + ProfilesDir = "{{.NodeHomeDir}}" + +[Gateway] + Level = "Info" + Port = 3{{ printf "%02d" .NodeNumber }}8 + [Gateway.Node] + Port = 3{{ printf "%02d" .NodeNumber }}7 + [Gateway.RateLimit] + Enabled = false + +[Metrics] + Level = "Info" + Timeout = "5s" + Port = 2{{ printf "%02d" (add .NodeNumber 10) }}2 + Enabled = true + +[Broker] + Level = "Info" + UseEventFile = false + PanicOnError = false + [Broker.SocketConfig] + Port = 3{{ printf "%02d" .NodeNumber }}5 + +[NetworkHistory] + Level = "Info" + Enabled = true + WipeOnStartup = true + AddSnapshotsToStore = true + AddSnapshotsInterval = "5s" + [NetworkHistory.Store] + IdSeed = "{{ b64enc "system-tests" }}-3479-439f-b7b7-{{.NodeNumber}}" + UseIpfsDefaultPeers = true + BootstrapPeers = [] + SwarmPort = 40{{.NodeNumber}}5 + StartWebUI = false + WebUIPort = 50{{.NodeNumber}}5 + [NetworkHistory.Snapshot] + PanicOnSnapshotCreationError = true + WaitForCreationLockTimeout = "5s" diff --git a/vega_sim/vegacapsule/templates/tendermint_full.tmpl b/vega_sim/vegacapsule/templates/tendermint_full.tmpl new file mode 100644 index 000000000..df06335d9 --- /dev/null +++ b/vega_sim/vegacapsule/templates/tendermint_full.tmpl @@ -0,0 +1,38 @@ +log_level = "info" + +proxy_app = "tcp://127.0.0.1:26{{ printf "%02d" (add .NodeNumber 60) }}}8" +moniker = "{{.TendermintNodePrefix}}-{{.NodeNumber}}" + +[rpc] + laddr = "tcp://0.0.0.0:26{{ printf "%02d" (add .NodeNumber 60) }}7" + unsafe = true + +[p2p] + laddr = "tcp://0.0.0.0:26{{ printf "%02d" (add .NodeNumber 60) }}6" + addr_book_strict = false + max_packet_msg_payload_size = 4096 + pex = false + allow_duplicate_ip = true + + persistent_peers = "{{- range $i, $peer := .NodePeers -}} + {{- if ne $i 0 }},{{end -}} + {{- $peer.ID}}@127.0.0.1:26{{ printf "%02d" (add $peer.Index 60) }}6 + {{- end -}}" + + persistent_peers_max_dial_period = "5s" + + +[mempool] + size = 10000 + cache_size = 20000 + +[consensus] + skip_timeout_commit = true + create_empty_blocks_interval = "500ms" + +[statesync] + enable = false + rpc_servers = "{{- range $i, $peer := .NodePeers -}} + {{- if ne $i 0 }},{{end -}} + 127.0.0.1:26{{ printf "%02d" (add $peer.Index 60) }}7 + {{- end -}}" \ No newline at end of file diff --git a/vega_sim/vegacapsule/templates/tendermint_validators.tmpl b/vega_sim/vegacapsule/templates/tendermint_validators.tmpl new file mode 100644 index 000000000..e563ffaee --- /dev/null +++ b/vega_sim/vegacapsule/templates/tendermint_validators.tmpl @@ -0,0 +1,38 @@ +log-level = "info" + +proxy_app = "tcp://127.0.0.1:26{{ printf "%02d" (add .NodeNumber 60) }}}8" +moniker = "{{.TendermintNodePrefix}}-{{.NodeNumber}}" + +[rpc] + laddr = "tcp://0.0.0.0:26{{ printf "%02d" (add .NodeNumber 60) }}7" + unsafe = true + +[p2p] + laddr = "tcp://0.0.0.0:26{{ printf "%02d" (add .NodeNumber 60) }}6" + addr_book_strict = false + max_packet_msg_payload_size = 4096 + pex = false + allow_duplicate_ip = true + + persistent_peers = "{{- range $i, $peer := .NodePeers -}} + {{- if ne $i 0 }},{{end -}} + {{- $peer.ID}}@127.0.0.1:26{{ printf "%02d" (add $peer.Index 60) }}6 + {{- end -}}" + + persistent_peers_max_dial_period = "5s" + + +[mempool] + size = 10000 + cache_size = 20000 + +[consensus] + skip_timeout_commit = true + create_empty_blocks_interval = "500ms" + +[statesync] + enable = false + rpc_servers = "{{- range $i, $peer := .NodePeers -}} + {{- if ne $i 0 }},{{end -}} + 127.0.0.1:26{{ printf "%02d" (add $peer.Index 60) }}7 + {{- end -}}" \ No newline at end of file diff --git a/vega_sim/vegacapsule/templates/vega_full.tmpl b/vega_sim/vegacapsule/templates/vega_full.tmpl new file mode 100644 index 000000000..508f88d98 --- /dev/null +++ b/vega_sim/vegacapsule/templates/vega_full.tmpl @@ -0,0 +1,138 @@ +[Admin] + [Admin.Server] + SocketPath = "{{.NodeHomeDir}}/vega.sock" + Enabled = true + +[API] + Level = "Info" + Port = 3{{ printf "%02d" .NodeNumber }}2 + [API.REST] + Port = 3{{ printf "%02d" .NodeNumber }}3 + +[Blockchain] + Level = "Info" + [Blockchain.Tendermint] + Level = "Info" + RPCAddr = "tcp://127.0.0.1:26{{ printf "%02d" (add .NodeNumber 60) }}7" + [Blockchain.Null] + Level = "Info" + Port = 3{{ printf "%02d" (add .NodeNumber 10) }}1 + +[Collateral] + Level = "Info" + +[CoreAPI] + LogLevel = "Info" + +[Execution] + Level = "Info" + [Execution.Matching] + Level = "Info" + [Execution.Risk] + Level = "Info" + [Execution.Position] + Level = "Info" + LogPositionUpdate = true + [Execution.Settlement] + Level = "Info" + [Execution.Fee] + Level = "Info" + [Execution.Liquidity] + Level = "Info" + +[EvtForward] + Level = "Info" + RetryRate = "1s" + [EvtForward.Ethereum] + Level = "Info" + PollEventRetryDuration = "1s" + +[Ethereum] + RPCEndpoint = "{{.ETHEndpoint}}" + RetryDelay = "1s" + +[NodeWallet] + Level = "Info" + [NodeWallet.ETH] + Level = "Info" + +[Processor] + Level = "Info" + [Processor.Ratelimit] + Requests = 10000 + PerNBlocks = 1 + +[Oracles] + Level = "Info" + +[Time] + Level = "Info" + +[Epoch] + Level = "Info" + +[Monitoring] + Level = "Info" + +[Metrics] + Level = "Info" + +[Governance] + Level = "Info" +[Assets] + Level = "Info" + +[Notary] + Level = "Info" + +[Genesis] + Level = "Info" + +[Validators] + Level = "Info" + +[Banking] + Level = "Info" + +[Stats] + Level = "Info" + +[NetworkParameters] + Level = "Info" + +[Limits] + Level = "Info" + +[Checkpoint] + Level = "Info" + +[Staking] + Level = "Info" + +[Broker] + Level = "Info" + [Broker.Socket] + Port = 3{{ printf "%02d" .NodeNumber }}5 + Enabled = true + [Broker.File] + Enabled = true + File = "{{.NodeHomeDir}}/eventlog.evt" + + +[Rewards] + Level = "Info" + +[Delegation] + Level = "Info" + +[Spam] + Level = "Info" + +[Snapshot] + Level = "Info" + +[StateVar] + Level = "Info" + +[Pprof] + Level = "Info" \ No newline at end of file diff --git a/vega_sim/vegacapsule/templates/vega_validators.tmpl b/vega_sim/vegacapsule/templates/vega_validators.tmpl new file mode 100644 index 000000000..dfb05c171 --- /dev/null +++ b/vega_sim/vegacapsule/templates/vega_validators.tmpl @@ -0,0 +1,130 @@ +[Admin] + [Admin.Server] + SocketPath = "{{.NodeHomeDir}}/vega.sock" + Enabled = true + +[API] + Level = "Info" + Port = 3{{ printf "%02d" .NodeNumber }}2 + [API.REST] + Port = 3{{ printf "%02d" .NodeNumber }}3 + +[Blockchain] + Level = "Info" + [Blockchain.Tendermint] + Level = "Info" + RPCAddr = "tcp://127.0.0.1:26{{ printf "%02d" (add .NodeNumber 60) }}7" + [Blockchain.Null] + Level = "Info" + Port = 3{{ printf "%02d" (add .NodeNumber 10) }}1 + +[Collateral] + Level = "Info" + +[CoreAPI] + LogLevel = "Info" + +[Execution] + Level = "Info" + [Execution.Matching] + Level = "Info" + [Execution.Risk] + Level = "Info" + [Execution.Position] + Level = "Info" + [Execution.Settlement] + Level = "Info" + [Execution.Fee] + Level = "Info" + [Execution.Liquidity] + Level = "Info" + +[Oracles] + Level = "Info" + +[Time] + Level = "Info" + +[Epoch] + Level = "Info" + +[Monitoring] + Level = "Info" + +[Metrics] + Level = "Info" + +[Governance] + Level = "Info" + +[Assets] + Level = "Info" + +[Notary] + Level = "Info" + +[EvtForward] + Level = "Info" + RetryRate = "1s" + {{if .FaucetPublicKey}} + BlockchainQueueAllowlist = ["{{ .FaucetPublicKey }}"] + {{end}} + [EvtForward.Ethereum] + Level = "Info" + PollEventRetryDuration = "1s" +[Genesis] + Level = "Info" + +[Validators] + Level = "Info" + +[Banking] + Level = "Info" + +[Stats] + Level = "Info" + +[NetworkParameters] + Level = "Info" + +[Limits] + Level = "Info" + +[Checkpoint] + Level = "Info" + +[Staking] + Level = "Info" + +[Ethereum] + RPCEndpoint = "{{.ETHEndpoint}}" + RetryDelay = "1s" + +[NodeWallet] + Level = "Info" + [NodeWallet.ETH] + Level = "Info" + +[Processor] + Level = "Info" + [Processor.Ratelimit] + Requests = 10000 + PerNBlocks = 1 + +[Broker] + Level = "Info" + +[Rewards] + Level = "Info" + +[Delegation] + Level = "Info" + +[Spam] + Level = "Info" + +[Snapshot] + Level = "Info" + +[StateVar] + Level = "Info" \ No newline at end of file diff --git a/vega_sim/wallet/vega_wallet.py b/vega_sim/wallet/vega_wallet.py index d0ea1761c..38793f333 100644 --- a/vega_sim/wallet/vega_wallet.py +++ b/vega_sim/wallet/vega_wallet.py @@ -123,6 +123,10 @@ def create_key(self, name: str, wallet_name: Optional[str] = None) -> None: wallet = ( wallet_name if wallet_name is not None else self.vega_default_wallet_name ) + + if wallet not in self.pub_keys: + self.create_wallet(name=wallet) + cmd = subprocess.run( [ self._wallet_path, @@ -190,6 +194,8 @@ def create_wallet(self, name: str) -> None: name, "--description", name, + "--output", + "json", ], capture_output=True, universal_newlines=True,