chore: migrate to using common protos repo (#679)

* chore: migrate to common proto repo * fix: run make export reqs * chore: poetry lock
vegaprotocol · Jun 11, 2024 · 6abf59f · 6abf59f
1 parent 45b8129
commit 6abf59f
Show file tree

Hide file tree

Showing 151 changed files with 158 additions and 34,934 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -14,7 +14,7 @@ pipeline {
         disableConcurrentBuilds(abortPrevious: true)
     }
     parameters {
-        string( name: 'VEGA_VERSION', defaultValue: 'e4890557c52ad5a24d14d38d64bfb79704c75cf7',
+        string( name: 'VEGA_VERSION', defaultValue: 'v0.76.8',
                 description: 'Git branch, tag or hash of the vegaprotocol/vega repository')
         string( name: 'VEGACAPSULE_VERSION', defaultValue: 'main',
                 description: 'Git branch, tag or hash of the vegaprotocol/vegacapsule repository')

diff --git a/Makefile b/Makefile
@@ -11,8 +11,6 @@ networks: pull_deps_networks build_deps_networks
 
 capsule: pull_deps_capsule build_deps_capsule
 
-proto: build_proto black
-
 clean_ui:
 	@echo "Deleting $(EXTERN_DIR)/console and ./vega_sim/bin/console" 
 	@rm -rf "$(EXTERN_DIR)/console"
@@ -87,12 +85,6 @@ build_deps_capsule:
 	@mkdir -p ./vega_sim/bin
 	cd ${EXTERN_DIR}/vegacapsule && go build -o ../../vega_sim/bin/ ./...
 
-build_proto: pull_deps
-	@rm -rf ./vega_sim/proto
-	@mkdir ./vega_sim/proto
-	@buf generate extern/vega/protos/sources 
-	@GENERATED_DIR=./vega_sim/proto scripts/post-generate.sh
-
 .PHONY: black
 black:
 	@black .

diff --git a/buf.gen.yaml b/buf.gen.yaml
diff --git a/examples/agent_market/agents.py b/examples/agent_market/agents.py
@@ -6,8 +6,8 @@
 from vega_sim.environment import StateAgent, VegaState
 from vega_sim.environment.agent import StateAgentWithWallet
 from vega_sim.null_service import VegaServiceNull
-from vega_sim.proto.vega import markets as markets_protos
-from vega_sim.proto.vega import vega as vega_protos, governance as gov_protos
+from vega_python_protos.protos.vega import markets as markets_protos
+from vega_python_protos.protos.vega import vega as vega_protos, governance as gov_protos
 from vega_sim.service import VegaService
 
 WalletConfig = namedtuple("WalletConfig", ["name", "passphrase"])

diff --git a/examples/notebooks/InvestigatePriceMove.ipynb b/examples/notebooks/InvestigatePriceMove.ipynb
@@ -11,7 +11,7 @@
     "import numpy as np\n",
     "\n",
     "import vega_sim.api.data_raw as data_raw\n",
-    "import vega_sim.proto.vega as vega_protos\n",
+    "import vega_python_protos.protos.vega as vega_protos\n",
     "from vega_sim.grpc.client import VegaTradingDataClientV2"
    ]
   },

diff --git a/examples/notebooks/feature_tests/0002-STTL-008.ipynb b/examples/notebooks/feature_tests/0002-STTL-008.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_sim.proto.vega as vega_protos\n"
+    "import vega_python_protos.protos.vega as vega_protos\n"
    ]
   },
   {

diff --git a/examples/notebooks/feature_tests/0004-AMND-010.ipynb b/examples/notebooks/feature_tests/0004-AMND-010.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_sim.proto.vega as vega_protos\n"
+    "import vega_python_protos.protos.vega as vega_protos\n"
    ]
   },
   {

diff --git a/examples/notebooks/feature_tests/0004-AMND-021.ipynb b/examples/notebooks/feature_tests/0004-AMND-021.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_sim.proto.vega as vega_protos"
+    "import vega_python_protos.protos.vega as vega_protos"
    ]
   },
   {

diff --git a/examples/notebooks/feature_tests/0004-AMND-022.ipynb b/examples/notebooks/feature_tests/0004-AMND-022.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_sim.proto.vega as vega_protos"
+    "import vega_python_protos.protos.vega as vega_protos"
    ]
   },
   {

diff --git a/examples/notebooks/feature_tests/0004-AMND-023.ipynb b/examples/notebooks/feature_tests/0004-AMND-023.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_sim.proto.vega as vega_protos"
+    "import vega_python_protos.protos.vega as vega_protos"
    ]
   },
   {

diff --git a/examples/notebooks/feature_tests/0004-AMND-024.ipynb b/examples/notebooks/feature_tests/0004-AMND-024.ipynb
@@ -9,7 +9,7 @@
     "from collections import namedtuple\n",
     "\n",
     "from vega_sim.null_service import VegaServiceNull\n",
-    "import vega_sim.proto.vega as vega_protos"
+    "import vega_python_protos.protos.vega as vega_protos"
    ]
   },
   {

diff --git a/examples/nullchain.py b/examples/nullchain.py
@@ -5,7 +5,7 @@
 
 from vega_sim.null_service import VegaServiceNull
 from vega_sim.service import MarketStateUpdateType
-import vega_sim.proto.vega as vega_protos
+import vega_python_protos.protos.vega as vega_protos
 
 
 WalletConfig = namedtuple("WalletConfig", ["name", "passphrase"])

diff --git a/examples/visualisations/utils.py b/examples/visualisations/utils.py
@@ -18,7 +18,7 @@
 from vega_sim.service import PeggedOrder
 from vega_sim.api.market import MarketConfig
 
-import vega_sim.proto.vega as vega_protos
+import vega_python_protos.protos.vega as vega_protos
 
 
 PartyConfig = namedtuple("WalletConfig", ["wallet_name", "key_name"])

diff --git a/generate_vega_init.py b/generate_vega_init.py
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -42,6 +42,7 @@ pettingzoo = {version = "*", optional = true}
 numba = {version = "^0.57.1", optional = true}
 tianshou = {version = "*", optional = true}
 protofuzz = { git = "https://github.com/trailofbits/protofuzz", branch="master" }
+vega_python_protos = {git = "https://github.com/vegaprotocol/vega-python-protos", branch="develop" }
 
 [tool.poetry.group.dev.dependencies]
 black = "*"

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -94,7 +94,8 @@ traitlets==5.14.3 ; python_version >= "3.10" and python_version < "3.12"
 typing-extensions==4.12.0 ; python_version >= "3.10" and python_version < "3.12"
 tzdata==2024.1 ; python_version >= "3.10" and python_version < "3.12"
 urllib3==2.2.1 ; python_version >= "3.10" and python_version < "3.12"
-vegapy @ git+https://github.com/cdummett/vegapy/@c5636a17f288b5fe9a5dbe8b16ebd2542ab3af15 ; python_version >= "3.10" and python_version < "3.12"
+vega-python-protos @ git+https://github.com/vegaprotocol/vega-python-protos@243d257709ee8484bd324d869233f1fcf8e22f6d ; python_version >= "3.10" and python_version < "3.12"
+vegapy @ git+https://github.com/cdummett/vegapy/@b2a7ae02a0adadd21ccae87688c976ff51d8cec6 ; python_version >= "3.10" and python_version < "3.12"
 wcwidth==0.2.13 ; python_version >= "3.10" and python_version < "3.12"
 websocket-client==1.8.0 ; python_version >= "3.10" and python_version < "3.12"
 wrapt==1.16.0 ; python_version >= "3.10" and python_version < "3.12"
diff --git a/requirements-learning.txt b/requirements-learning.txt
@@ -130,7 +130,8 @@ triton==2.3.0 ; platform_system == "Linux" and platform_machine == "x86_64" and
 typing-extensions==4.12.0 ; python_version >= "3.10" and python_version < "3.12"
 tzdata==2024.1 ; python_version >= "3.10" and python_version < "3.12"
 urllib3==2.2.1 ; python_version >= "3.10" and python_version < "3.12"
-vegapy @ git+https://github.com/cdummett/vegapy/@c5636a17f288b5fe9a5dbe8b16ebd2542ab3af15 ; python_version >= "3.10" and python_version < "3.12"
+vega-python-protos @ git+https://github.com/vegaprotocol/vega-python-protos@243d257709ee8484bd324d869233f1fcf8e22f6d ; python_version >= "3.10" and python_version < "3.12"
+vegapy @ git+https://github.com/cdummett/vegapy/@b2a7ae02a0adadd21ccae87688c976ff51d8cec6 ; python_version >= "3.10" and python_version < "3.12"
 wcwidth==0.2.13 ; python_version >= "3.10" and python_version < "3.12"
 websocket-client==1.8.0 ; python_version >= "3.10" and python_version < "3.12"
 werkzeug==3.0.3 ; python_version >= "3.10" and python_version < "3.12"

diff --git a/requirements.txt b/requirements.txt
@@ -1,30 +1,42 @@
+black==24.4.2 ; python_version >= "3.10" and python_version < "3.12"
 certifi==2024.2.2 ; python_version >= "3.10" and python_version < "3.12"
 cffi==1.16.0 ; python_version >= "3.10" and python_version < "3.12"
 charset-normalizer==3.3.2 ; python_version >= "3.10" and python_version < "3.12"
+click==8.1.7 ; python_version >= "3.10" and python_version < "3.12"
+colorama==0.4.6 ; python_version >= "3.10" and python_version < "3.12" and (sys_platform == "win32" or platform_system == "Windows")
 contourpy==1.2.1 ; python_version >= "3.10" and python_version < "3.12"
 cycler==0.12.1 ; python_version >= "3.10" and python_version < "3.12"
 deprecated==1.2.14 ; python_version >= "3.10" and python_version < "3.12"
 docker==7.1.0 ; python_version >= "3.10" and python_version < "3.12"
+exceptiongroup==1.2.1 ; python_version >= "3.10" and python_version < "3.11"
 fonttools==4.52.4 ; python_version >= "3.10" and python_version < "3.12"
 googleapis-common-protos==1.63.0 ; python_version >= "3.10" and python_version < "3.12"
 grpcio-tools==1.62.2 ; python_version >= "3.10" and python_version < "3.12"
 grpcio==1.64.0 ; python_version >= "3.10" and python_version < "3.12"
 idna==3.7 ; python_version >= "3.10" and python_version < "3.12"
 inflection==0.5.1 ; python_version >= "3.10" and python_version < "3.12"
+iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "3.12"
+jinja2==3.1.4 ; python_version >= "3.10" and python_version < "3.12"
 kiwisolver==1.4.5 ; python_version >= "3.10" and python_version < "3.12"
+markupsafe==2.1.5 ; python_version >= "3.10" and python_version < "3.12"
 matplotlib==3.9.0 ; python_version >= "3.10" and python_version < "3.12"
+mypy-extensions==1.0.0 ; python_version >= "3.10" and python_version < "3.12"
 numpy==1.24.4 ; python_version >= "3.10" and python_version < "3.12"
 packaging==24.0 ; python_version >= "3.10" and python_version < "3.12"
 pandas==2.2.2 ; python_version >= "3.10" and python_version < "3.12"
+pathspec==0.12.1 ; python_version >= "3.10" and python_version < "3.12"
 pillow==10.3.0 ; python_version >= "3.10" and python_version < "3.12"
+platformdirs==4.2.2 ; python_version >= "3.10" and python_version < "3.12"
 plotly==5.22.0 ; python_version >= "3.10" and python_version < "3.12"
+pluggy==1.5.0 ; python_version >= "3.10" and python_version < "3.12"
 protobuf==4.25.3 ; python_version >= "3.10" and python_version < "3.12"
 protoc-gen-openapiv2==0.0.1 ; python_version >= "3.10" and python_version < "3.12"
 protofuzz @ git+https://github.com/trailofbits/protofuzz@a4fd0936ae501dba0fc89d3af4ead8ef3aaad1d1 ; python_version >= "3.10" and python_version < "3.12"
 psutil==5.9.8 ; python_version >= "3.10" and python_version < "3.12"
 pycparser==2.22 ; python_version >= "3.10" and python_version < "3.12"
 pynacl==1.5.0 ; python_version >= "3.10" and python_version < "3.12"
 pyparsing==3.1.2 ; python_version >= "3.10" and python_version < "3.12"
+pytest==7.4.4 ; python_version >= "3.10" and python_version < "3.12"
 python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "3.12"
 python-dotenv==1.0.1 ; python_version >= "3.10" and python_version < "3.12"
 pytz==2024.1 ; python_version >= "3.10" and python_version < "3.12"
@@ -35,7 +47,10 @@ setuptools==70.0.0 ; python_version >= "3.10" and python_version < "3.12"
 six==1.16.0 ; python_version >= "3.10" and python_version < "3.12"
 tenacity==8.3.0 ; python_version >= "3.10" and python_version < "3.12"
 toml==0.10.2 ; python_version >= "3.10" and python_version < "3.12"
+tomli==2.0.1 ; python_version >= "3.10" and python_version < "3.11"
+typing-extensions==4.12.0 ; python_version >= "3.10" and python_version < "3.11"
 tzdata==2024.1 ; python_version >= "3.10" and python_version < "3.12"
 urllib3==2.2.1 ; python_version >= "3.10" and python_version < "3.12"
+vega-python-protos @ git+https://github.com/vegaprotocol/vega-python-protos@243d257709ee8484bd324d869233f1fcf8e22f6d ; python_version >= "3.10" and python_version < "3.12"
 websocket-client==1.8.0 ; python_version >= "3.10" and python_version < "3.12"
 wrapt==1.16.0 ; python_version >= "3.10" and python_version < "3.12"
diff --git a/tests/integration/test_governance.py b/tests/integration/test_governance.py
@@ -1,6 +1,6 @@
 import pytest
 
-import vega_sim.proto.vega as vega_protos
+import vega_python_protos.protos.vega as vega_protos
 
 from vega_sim.null_service import VegaServiceNull
 from tests.integration.utils.fixtures import WalletConfig, vega_service

diff --git a/tests/integration/test_liquidation.py b/tests/integration/test_liquidation.py
@@ -1,5 +1,5 @@
 import pytest
-from vega_sim import proto as protos
+from vega_python_protos import protos
 from vega_sim.null_service import VegaServiceNull
 from tests.integration.utils.fixtures import (
     ASSET_NAME,

diff --git a/tests/integration/test_trading.py b/tests/integration/test_trading.py
@@ -1,5 +1,5 @@
 import pytest
-import vega_sim.proto.vega as vega_protos
+import vega_python_protos.protos.vega as vega_protos
 from vega_sim.api.market import MarketConfig
 import vega_sim.builders as build
 

diff --git a/tests/integration/test_update_market.py b/tests/integration/test_update_market.py
@@ -9,9 +9,9 @@
 )
 from vega_sim.null_service import VegaServiceNull
 from vega_sim.service import MarketStateUpdateType
-import vega_sim.proto.vega as vega_protos
-import vega_sim.proto.vega.data.v1 as oracles_protos
-import vega_sim.proto.vega.data_source_pb2 as data_source_protos
+import vega_python_protos.protos.vega as vega_protos
+import vega_python_protos.protos.vega.data.v1 as oracles_protos
+import vega_python_protos.protos.vega.data_source_pb2 as data_source_protos
 
 
 LIQ = WalletConfig("liq", "liq")

diff --git a/tests/vega_sim/api/test_api_coverage.py b/tests/vega_sim/api/test_api_coverage.py
@@ -4,8 +4,8 @@
 import pytest
 import re
 
-import vega_sim.proto.data_node.api.v2 as data_node_protos_v2
-import vega_sim.proto.vega as vega_protos
+import vega_python_protos.protos.data_node.api.v2 as data_node_protos_v2
+import vega_python_protos.protos.vega as vega_protos
 import vega_sim.api.data as data
 import vega_sim.api.data_raw as data_raw