feat: merge protos into Vega and remove datanode (#105)

* feat: Update protos and remove data-node

Makefile

@@ -1,14 +1,12 @@
 VEGA_TAG := develop
-DATA_NODE_TAG := develop
 CONSOLE_TAG := master
-PROTO_TAG := develop
 EXTERN_DIR := "./extern"
 
 all: pull_deps build_deps
 
 ui: pull_deps_ui build_deps_ui
 
-proto: pull_deps_proto build_proto black
+proto: build_proto black
 
 pull_deps:
 	@if [ ! -d ./extern/ ]; then mkdir ./extern/; fi
@@ -20,19 +18,10 @@ ifneq (${VEGA_TAG},develop)
 else
 	@git -C ${EXTERN_DIR}/vega checkout develop; git -C ${EXTERN_DIR}/vega pull
 endif
-	@echo "Downloading data-node"
-	@if [ ! -d ./extern/data-node ]; then mkdir ./extern/data-node; git clone https://github.com/vegaprotocol/data-node ${EXTERN_DIR}/data-node; fi
-ifneq (${DATA_NODE_TAG},develop)
-	@git -C ${EXTERN_DIR}/data-node pull; git -C ${EXTERN_DIR}/data-node checkout ${DATA_NODE_TAG}
-else
-	@git -C ${EXTERN_DIR}/data-node checkout develop; git -C ${EXTERN_DIR}/data-node pull
-endif
-
 
 build_deps:
 	@mkdir -p ./vega_sim/bin
 	cd ${EXTERN_DIR}/vega && go build -o ../../vega_sim/bin/ ./...
-	cd ${EXTERN_DIR}/data-node && go build -o ../../vega_sim/bin/ ./...
 
 pull_deps_ui:
 	@if [ ! -d ./extern/ ]; then mkdir ./extern/; fi
@@ -50,21 +39,10 @@ build_deps_ui:
 	@rsync -av ${EXTERN_DIR}/console vega_sim/bin/ --exclude ${EXTERN_DIR}/console/.git
 	@yarn --cwd vega_sim/bin/console install
 
-pull_deps_proto:
-	@if [ ! -d ./extern/ ]; then mkdir ./extern/; fi
-	@echo "Downloading Git dependencies into " ${EXTERN_DIR}
-	@echo "Downloading Vega Protos"
-	@if [ ! -d ./extern/proto ]; then mkdir ${EXTERN_DIR}/proto; git clone https://github.com/vegaprotocol/protos ${EXTERN_DIR}/proto; fi
-ifneq (${PROTO_TAG},develop)
-	@git -C ${EXTERN_DIR}/proto pull; git -C ${EXTERN_DIR}/proto checkout ${PROTO_TAG}
-else
-	@git -C ${EXTERN_DIR}/proto checkout develop; git -C ${EXTERN_DIR}/proto pull
-endif
-
-build_proto:
+build_proto: pull_deps
 	@rm -rf ./vega_sim/proto
 	@mkdir ./vega_sim/proto
-	@python -m grpc_tools.protoc -I ${EXTERN_DIR}/proto/sources --python_out=vega_sim/proto --grpc_python_out=vega_sim/proto $(shell find ${EXTERN_DIR}/proto/sources -name '*.proto')
+	@python -m grpc_tools.protoc -I ${EXTERN_DIR}/vega/protos/sources --python_out=vega_sim/proto --grpc_python_out=vega_sim/proto $(shell find ${EXTERN_DIR}/vega/protos/sources -name '*.proto')
 	@GENERATED_DIR=./vega_sim/proto scripts/post-generate.sh
 
 .PHONY: black

examples/nullchain.py

@@ -28,7 +28,7 @@
     logging.basicConfig(level=logging.INFO)
 
     with VegaServiceNull(
-        run_with_console=False,
+        run_with_console=True,
         start_order_feed=True,
     ) as vega:
         for wallet in wallets:

vega_sim/api/governance.py

@@ -6,6 +6,7 @@
 import vega_sim.grpc.client as vac
 import vega_sim.proto.data_node.api.v1 as data_node_protos
 import vega_sim.proto.vega as vega_protos
+from vega_sim.proto.vega.commands.v1.commands_pb2 import ProposalSubmission
 import vega_sim.proto.vega.oracles.v1 as oracles_protos
 import vega_sim.proto.vega.commands.v1 as commands_protos
 from vega_sim.api.helpers import (
@@ -267,14 +268,13 @@ def propose_future_market(
     )
     proposal.terms.new_market.CopyFrom(market_proposal)
 
-    _make_and_wait_for_proposal(
+    return _make_and_wait_for_proposal(
         wallet_name=wallet_name,
         wallet=wallet,
         proposal=proposal,
         data_client=data_client,
         time_forward_fn=time_forward_fn,
-    )
-    return proposal.reference
+    ).proposal.id
 
 
 def propose_network_parameter_change(
@@ -300,14 +300,13 @@ def propose_network_parameter_change(
             changes=vega_protos.vega.NetworkParameter(key=parameter, value=value)
         )
     )
-    _make_and_wait_for_proposal(
+    return _make_and_wait_for_proposal(
         wallet_name=wallet_name,
         wallet=wallet,
         proposal=network_param_update,
         data_client=data_client,
         time_forward_fn=time_forward_fn,
-    )
-    return network_param_update.reference
+    ).proposal.id
 
 
 def propose_market_update(
@@ -320,7 +319,7 @@ def propose_market_update(
     validation_time: Optional[int] = None,
     data_client: Optional[vac.VegaTradingDataClient] = None,
     time_forward_fn: Optional[Callable[[], None]] = None,
-):
+) -> str:
     network_param_update = _build_generic_proposal(
         pub_key=wallet.public_key(wallet_name),
         data_client=data_client,
@@ -332,14 +331,13 @@ def propose_market_update(
         vega_protos.governance.UpdateMarket(market_id=market_id, changes=market_update)
     )
 
-    _make_and_wait_for_proposal(
+    return _make_and_wait_for_proposal(
         wallet_name=wallet_name,
         wallet=wallet,
         proposal=network_param_update,
         data_client=data_client,
         time_forward_fn=time_forward_fn,
-    )
-    return network_param_update.reference
+    ).proposal.id
 
 
 def approve_proposal(
@@ -370,7 +368,7 @@ def propose_asset(
     enactment_time: Optional[int] = None,
     validation_time: Optional[int] = None,
     time_forward_fn: Optional[Callable[[], None]] = None,
-):
+) -> str:
     asset_detail = vega_protos.assets.AssetDetails(
         name=name,
         symbol=symbol,
@@ -391,14 +389,13 @@ def propose_asset(
     proposal.terms.new_asset.CopyFrom(
         vega_protos.governance.NewAsset(changes=asset_detail)
     )
-    _make_and_wait_for_proposal(
+    return _make_and_wait_for_proposal(
         wallet_name=wallet_name,
         wallet=wallet,
         proposal=proposal,
         data_client=data_client,
         time_forward_fn=time_forward_fn,
-    )
-    return proposal.reference
+    ).proposal.id
 
 
 def _build_generic_proposal(
@@ -448,7 +445,7 @@ def _make_and_wait_for_proposal(
     proposal: commands_protos.commands.ProposalSubmission,
     data_client: vac.VegaTradingDataClient,
     time_forward_fn: Optional[Callable[[], None]] = None,
-):
+) -> ProposalSubmission:
     wallet.submit_transaction(
         transaction=proposal, name=wallet_name, transaction_type="proposal_submission"
     )
@@ -469,6 +466,7 @@ def _make_and_wait_for_proposal(
             f" {enum_to_str(vega_protos.governance.ProposalError, proposal.proposal.reason)}."
             f" Any further info: {proposal.proposal.error_details}"
         )
+    return proposal
 
 
 def settle_oracle(

vega_sim/api/trading.py

@@ -207,16 +207,11 @@ def amend_order(
         size_delta=volume_delta,
         time_in_force=time_in_force,
     )
-    for name, val in [
-        ("price", vega_protos.vega.Price(value=str(price))),
-    ]:
-        if val is not None:
-            getattr(order_data, name).CopyFrom(val)
-
     for name, val in [
         ("expires_at", expires_at),
         ("pegged_offset", pegged_offset),
         ("pegged_reference", pegged_reference),
+        ("price", str(price) if price is not None else None),
     ]:
         if val is not None:
             setattr(order_data, name, val)
@@ -379,7 +374,6 @@ def build_new_market_commitment(
     buy_specs: List[Tuple[str, int, int]],
     sell_specs: List[Tuple[str, int, int]],
 ) -> vega_protos.governance.NewMarketCommitment:
-
     return vega_protos.governance.NewMarketCommitment(
         commitment_amount=str(commitment_amount),
         fee=str(fee),

vega_sim/null_service.py

@@ -321,11 +321,18 @@ def manage_vega_processes(
     run_wallet: bool = False,
     retain_log_files: bool = False,
 ) -> None:
+    logging.basicConfig(level=logging.INFO)
     port_config = port_config if port_config is not None else {}
 
     # Explicitly not using context here so that crashed logs are retained
     tmp_vega_dir = tempfile.mkdtemp()
     logger.info(f"Running NullChain from vegahome of {tmp_vega_dir}")
+    if port_config.get(Ports.DATA_NODE_GRAPHQL):
+        logger.info(
+            f"Launching GraphQL node at port {port_config.get(Ports.DATA_NODE_GRAPHQL)}"
+        )
+    if port_config.get(Ports.CONSOLE):
+        logger.info(f"Launching Console at port {port_config.get(Ports.CONSOLE)}")
     shutil.copytree(vega_home_path, f"{tmp_vega_dir}/vegahome")
 
     tmp_vega_home = tmp_vega_dir + "/vegahome"
@@ -553,7 +560,7 @@ def _generate_port_config(self) -> Dict[Ports, int]:
         }
 
     def start(self, block_on_startup: bool = True) -> None:
-        ctx = multiprocessing.get_context("fork")
+        ctx = multiprocessing.get_context()
         self.proc = ctx.Process(
             target=manage_vega_processes,
             kwargs={

vega_sim/parameter_test/parameter/experiment.py

@@ -53,7 +53,9 @@ def _run_parameter_iteration(
     value: str,
     additional_parameters_to_set: Optional[Dict[str, str]] = None,
 ) -> Any:
-    with VegaServiceNull(warn_on_raw_data_access=False, retain_log_files=True) as vega:
+    with VegaServiceNull(
+        warn_on_raw_data_access=False, retain_log_files=True, run_with_console=False
+    ) as vega:
         vega.create_wallet(*PARAMETER_AMEND_WALLET)
         vega.mint(
             PARAMETER_AMEND_WALLET[0],
@@ -71,7 +73,11 @@ def _run_parameter_iteration(
             PARAMETER_AMEND_WALLET[0], parameter=parameter_to_vary, new_value=value
         )
 
-        return scenario.run_iteration(vega=vega)
+        res = scenario.run_iteration(vega=vega)
+        # import pdb
+
+        # pdb.set_trace()
+        return res
 
 
 def run_single_parameter_experiment(

vega_sim/parameter_test/run.py

@@ -12,9 +12,6 @@
 
     args = parser.parse_args()
     logging.basicConfig(level=logging.INFO)
-
-    logging.basicConfig(level=logging.INFO)
-
     config_map = {c.name: c for c in CONFIGS}
 
     experiment_to_run = config_map[args.config]