From eb0d3766fbe23dba13c0214676534f9dd628bcbb Mon Sep 17 00:00:00 2001 From: F4ever Date: Fri, 24 Jan 2025 11:07:17 +0100 Subject: [PATCH 1/8] reformat contants.py --- src/constants.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/constants.py b/src/constants.py index 16a717f7c..ec7495d27 100644 --- a/src/constants.py +++ b/src/constants.py @@ -12,8 +12,11 @@ # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#rewards-and-penalties PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX = 3 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#gwei-values -EFFECTIVE_BALANCE_INCREMENT = 2**0 * 10**9 +EFFECTIVE_BALANCE_INCREMENT = Gwei(2**0 * 10**9) MAX_EFFECTIVE_BALANCE = Gwei(32 * 10**9) +# https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#gwei-values +MAX_EFFECTIVE_BALANCE_ELECTRA = Gwei(2**11 * 10**9) +MIN_ACTIVATION_BALANCE = Gwei(2**5 * 10**9) # https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#execution MAX_WITHDRAWALS_PER_PAYLOAD = 2**4 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#withdrawal-prefixes @@ -32,17 +35,12 @@ # https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#withdrawals-processing MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP = 2**3 -# https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#gwei-values -MIN_ACTIVATION_BALANCE = Gwei(2**5 * 10**9) -MAX_EFFECTIVE_BALANCE_ELECTRA = Gwei(2**11 * 10**9) - +# Lido contracts constants LIDO_DEPOSIT_AMOUNT = MIN_ACTIVATION_BALANCE - -# Local constants -GWEI_TO_WEI = 10**9 SHARE_RATE_PRECISION_E27 = 10**27 TOTAL_BASIS_POINTS = 10000 +# Local constants +GWEI_TO_WEI = 10**9 MAX_BLOCK_GAS_LIMIT = 30_000_000 - UINT64_MAX = 2**64 - 1 From f539d58cd661637836cf1ecc0d4543a6b1fc4767 Mon Sep 17 00:00:00 2001 From: F4ever Date: Fri, 24 Jan 2025 11:07:39 +0100 Subject: [PATCH 2/8] Add additional CSM var check --- src/main.py | 4 ++-- src/variables.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main.py b/src/main.py index b6b2a7c7d..a89e66e34 100644 --- a/src/main.py +++ b/src/main.py @@ -164,12 +164,12 @@ def ipfs_providers() -> Iterator[IPFSProvider]: raise ValueError(msg) module = OracleModule(module_name_arg) - if module == OracleModule.CHECK: + if module is OracleModule.CHECK: errors = variables.check_uri_required_variables() variables.raise_from_errors(errors) sys.exit(check()) - errors = variables.check_all_required_variables() + errors = variables.check_all_required_variables(module) variables.raise_from_errors(errors) main(module) diff --git a/src/variables.py b/src/variables.py index 942561c2f..f94783d25 100644 --- a/src/variables.py +++ b/src/variables.py @@ -4,6 +4,7 @@ from eth_account import Account +from src.types import OracleModule from src.utils.env import from_file_or_env # - Providers- @@ -85,10 +86,14 @@ CACHE_PATH: Final = Path(os.getenv("CACHE_PATH", ".")) -def check_all_required_variables(): +def check_all_required_variables(module: OracleModule): errors = check_uri_required_variables() if not LIDO_LOCATOR_ADDRESS: errors.append('LIDO_LOCATOR_ADDRESS') + + if module is OracleModule.CSM and not CSM_MODULE_ADDRESS: + errors.append('CSM_MODULE_ADDRESS') + return errors From 1cf4f98b20cc2702bab3079df863921e0d4340ab Mon Sep 17 00:00:00 2001 From: F4ever Date: Fri, 24 Jan 2025 11:31:29 +0100 Subject: [PATCH 3/8] Upgrade check version compatibility --- src/modules/submodules/consensus.py | 24 ++++++---- src/modules/submodules/exceptions.py | 4 ++ src/modules/submodules/oracle_module.py | 19 +++++--- .../submodules/consensus/test_consensus.py | 46 +++++++++++++++---- 4 files changed, 69 insertions(+), 24 deletions(-) diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index 29807c1ad..7e8ead9ca 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -233,31 +233,37 @@ def get_blockstamp_for_report(self, last_finalized_blockstamp: BlockStamp) -> Re return bs - def _check_contract_versions(self, blockstamp: ReferenceBlockStamp): + def _check_compatability(self, blockstamp: BlockStamp): """ Check if Oracle can process report on reference blockstamp. """ - self._check_compatability(blockstamp.block_hash) - self._check_compatability('latest') - - def _check_compatability(self, block_tag: BlockIdentifier): - contract_version = self.report_contract.get_contract_version(block_tag) - consensus_version = self.report_contract.get_consensus_version(block_tag) + contract_version = self.report_contract.get_contract_version(blockstamp.block_hash) + consensus_version = self.report_contract.get_consensus_version(blockstamp.block_hash) compatibility = (contract_version, consensus_version) in self.COMPATIBLE_ONCHAIN_VERSIONS if not compatibility: raise IncompatibleOracleVersion( - f'Incompatible Oracle version. Block tag: {repr(block_tag)}. ' + f'Incompatible Oracle version. Block tag: {repr(blockstamp.block_hash)}. ' f'Expected (Contract, Consensus) versions: {', '.join(repr(v) for v in self.COMPATIBLE_ONCHAIN_VERSIONS)}, ' f'Got ({contract_version}, {consensus_version})' ) + contract_version_latest = self.report_contract.get_contract_version('latest') + consensus_version_latest = self.report_contract.get_consensus_version('latest') + + if not (contract_version == contract_version_latest and consensus_version == consensus_version_latest): + raise ContractVersionMismatch( + 'The Oracle can\'t process the report on the reference blockstamp. ' + f'The Contract or Consensus versions differ between the latest and {blockstamp.block_hash}, ' + 'further processing report can lead to unexpected behavior.' + ) + # ----- Working with report ----- def process_report(self, blockstamp: ReferenceBlockStamp) -> None: """Builds and sends report for current frame with provided blockstamp.""" # Make sure module is compatible with contracts on reference and latest blockstamps. - self._check_contract_versions(blockstamp) + self._check_compatability(blockstamp) report_data = self.build_report(blockstamp) logger.info({'msg': 'Build report.', 'value': report_data}) diff --git a/src/modules/submodules/exceptions.py b/src/modules/submodules/exceptions.py index aadc55c10..82d5c8d95 100644 --- a/src/modules/submodules/exceptions.py +++ b/src/modules/submodules/exceptions.py @@ -4,3 +4,7 @@ class IsNotMemberException(Exception): class IncompatibleOracleVersion(Exception): pass + + +class ContractVersionMismatch(Exception): + pass diff --git a/src/modules/submodules/oracle_module.py b/src/modules/submodules/oracle_module.py index c6cab4782..d2bd42241 100644 --- a/src/modules/submodules/oracle_module.py +++ b/src/modules/submodules/oracle_module.py @@ -11,7 +11,7 @@ from src.metrics.healthcheck_server import pulse from src.metrics.prometheus.basic import ORACLE_BLOCK_NUMBER, ORACLE_SLOT_NUMBER -from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion +from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch from src.providers.http_provider import NotOkResponse from src.providers.ipfs import IPFSError from src.providers.keys.client import KeysOutdatedException @@ -80,14 +80,19 @@ def _cycle(self): self.refresh_contracts_if_address_change() self.run_cycle(blockstamp) - except IsNotMemberException as exception: + except IsNotMemberException as error: logger.error({'msg': 'Provided account is not part of Oracle`s committee.'}) - raise exception - except IncompatibleOracleVersion as exception: + raise error + except IncompatibleOracleVersion as error: logger.error({'msg': 'Incompatible Contract version. Please update Oracle Daemon.'}) - raise exception - except DecoratorTimeoutError as exception: - logger.error({'msg': 'Oracle module do not respond.', 'error': str(exception)}) + raise error + except ContractVersionMismatch as error: + logger.error({ + 'msg': 'The oracle can\'t submit a report, because the contract\'s consensus version has changed.', + 'error': str(error), + }) + except DecoratorTimeoutError as error: + logger.error({'msg': 'Oracle module do not respond.', 'error': str(error)}) except NoActiveProviderError as error: logger.error({'msg': ''.join(traceback.format_exception(error))}) except RequestsConnectionError as error: diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index 20ddb35fc..0eaa10395 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -2,12 +2,13 @@ from unittest.mock import Mock import pytest +from toolz.functoolz import return_none from web3.exceptions import ContractCustomError from src import variables from src.modules.submodules import consensus as consensus_module from src.modules.submodules.consensus import ZERO_HASH, ConsensusModule, IsNotMemberException, MemberInfo -from src.modules.submodules.exceptions import IncompatibleOracleVersion +from src.modules.submodules.exceptions import IncompatibleOracleVersion, ContractVersionMismatch from src.modules.submodules.types import ChainConfig from src.providers.consensus.types import BeaconSpecResponse from src.types import BlockStamp, ReferenceBlockStamp @@ -193,15 +194,44 @@ def test_get_blockstamp_for_report_slot_deadline_missed(web3, consensus, caplog, @pytest.mark.unit -def test_compatible_contract_version(consensus): +@pytest.mark.parametrize( + 'contract_version,consensus_version', + [ + pytest.param(1, 2, marks=pytest.mark.xfail(raises=IncompatibleOracleVersion, strict=True)), + pytest.param(3, 3, marks=pytest.mark.xfail(raises=IncompatibleOracleVersion, strict=True)), + pytest.param(2, 1, marks=pytest.mark.xfail(raises=IncompatibleOracleVersion, strict=True)), + (2, 2), + ], +) +def test_incompatible_oracle(consensus, contract_version, consensus_version): bs = ReferenceBlockStampFactory.build() - consensus._check_compatability = Mock() - consensus._check_contract_versions(bs) + consensus.report_contract.get_contract_version = Mock(return_value=contract_version) + consensus.report_contract.get_consensus_version = Mock(return_value=consensus_version) + + consensus._check_compatability(bs) + + +@pytest.mark.unit +@pytest.mark.parametrize( + 'contract_version,consensus_version', + [ + pytest.param(3, 2, marks=pytest.mark.xfail(raises=ContractVersionMismatch, strict=True)), + pytest.param(3, 3, marks=pytest.mark.xfail(raises=ContractVersionMismatch, strict=True)), + pytest.param(2, 3, marks=pytest.mark.xfail(raises=ContractVersionMismatch, strict=True)), + (2, 2), + ], +) +def test_contract_upgrade_before_report_submited(consensus, contract_version, consensus_version): + bs = ReferenceBlockStampFactory.build() + + check_latest_contract = lambda tag: contract_version if tag == 'latest' else 2 + consensus.report_contract.get_contract_version = Mock(side_effect=check_latest_contract) + + check_latest_consensus = lambda tag: consensus_version if tag == 'latest' else 2 + consensus.report_contract.get_consensus_version = Mock(side_effect=check_latest_consensus) - assert consensus._check_compatability.call_count == 2 - assert consensus._check_compatability._mock_mock_calls[0].args[0] == bs.block_hash - assert consensus._check_compatability._mock_mock_calls[1].args[0] == 'latest' + consensus._check_compatability(bs) @pytest.mark.unit @@ -212,7 +242,7 @@ def test_incompatible_contract_version(consensus): consensus.report_contract.get_consensus_version = Mock(return_value=1) with pytest.raises(IncompatibleOracleVersion): - consensus._check_contract_versions(bs) + consensus._check_compatability(bs) @pytest.mark.unit From 5a700c2bf98255b2304117f34631368f8ad3a1c3 Mon Sep 17 00:00:00 2001 From: F4ever Date: Fri, 24 Jan 2025 11:33:45 +0100 Subject: [PATCH 4/8] refactor get submit member --- src/modules/submodules/consensus.py | 18 ++++++------------ .../submodules/consensus/test_consensus.py | 3 ++- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index 7e8ead9ca..ffcccbba2 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -19,7 +19,7 @@ FRAME_DEADLINE_SLOT, ORACLE_MEMBER_INFO ) -from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion +from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch from src.modules.submodules.types import ChainConfig, MemberInfo, ZERO_HASH, CurrentFrame, FrameConfig from src.utils.blockstamp import build_blockstamp from src.utils.web3converter import Web3Converter @@ -161,7 +161,11 @@ def get_member_info(self, blockstamp: BlockStamp) -> MemberInfo: if revert.data != InitialEpochIsYetToArriveRevert: raise revert - is_submit_member = self._is_submit_member(blockstamp) + is_submit_member = self.report_contract.has_role( + self.report_contract.submit_data_role(blockstamp.block_hash), + variables.ACCOUNT.address, + blockstamp.block_hash + ) if not is_member and not is_submit_member: raise IsNotMemberException( @@ -184,16 +188,6 @@ def get_member_info(self, blockstamp: BlockStamp) -> MemberInfo: return mi - def _is_submit_member(self, blockstamp: BlockStamp) -> bool: - if not variables.ACCOUNT: - return True - - return self.report_contract.has_role( - self.report_contract.submit_data_role(blockstamp.block_hash), - variables.ACCOUNT.address, - blockstamp.block_hash - ) - # ----- Calculation reference slot for report ----- def get_blockstamp_for_report(self, last_finalized_blockstamp: BlockStamp) -> ReferenceBlockStamp | None: """ diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index 0eaa10395..ada60f488 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -163,7 +163,8 @@ def test_first_frame_is_not_yet_started(web3, consensus, caplog, use_account): get_current_frame=Mock(side_effect=err), get_consensus_state_for_member=Mock(side_effect=err) ) consensus._get_consensus_contract = Mock(return_value=consensus_contract) - consensus._is_submit_member = Mock(return_value=True) + consensus.report_contract.submit_data_role = Mock(return_value='0x0') + consensus.report_contract.has_role = Mock(return_value=True) consensus.get_frame_config = Mock(return_value=FrameConfigFactory.build(initial_epoch=5, epochs_per_frame=10)) consensus.get_chain_config = Mock(return_value=ChainConfigFactory.build()) From e6a5536806bebc889ce811acc0dd0d84d04fb52d Mon Sep 17 00:00:00 2001 From: F4ever Date: Fri, 24 Jan 2025 11:39:07 +0100 Subject: [PATCH 5/8] more refactoring --- src/modules/submodules/consensus.py | 1 - src/services/exit_order/iterator_state.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index ffcccbba2..152a94c20 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -4,7 +4,6 @@ from typing import cast from eth_abi import encode -from eth_typing import BlockIdentifier from hexbytes import HexBytes from web3.exceptions import ContractCustomError diff --git a/src/services/exit_order/iterator_state.py b/src/services/exit_order/iterator_state.py index fda17a37e..a8decbc78 100644 --- a/src/services/exit_order/iterator_state.py +++ b/src/services/exit_order/iterator_state.py @@ -81,7 +81,7 @@ def prepare_lido_node_operator_stats( operator_predictable_stats[global_index] = NodeOperatorPredictableState( predictable_validators_total_age, transient_validators_count + predictable_validators_count, - operator.is_target_limit_active, + bool(operator.is_target_limit_active), operator.target_validators_count, max(0, delayed_validators_count - operator.refunded_validators_count) ) From 8556dd71b8c0b599162792d6db829cdd60d4be19 Mon Sep 17 00:00:00 2001 From: F4ever Date: Fri, 24 Jan 2025 11:48:15 +0100 Subject: [PATCH 6/8] Move check balance to more appropriate method --- src/modules/submodules/consensus.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index 152a94c20..02ecf152c 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -136,6 +136,10 @@ def get_member_info(self, blockstamp: BlockStamp) -> MemberInfo: current_frame_consensus_report = current_frame_member_report = ZERO_HASH if variables.ACCOUNT: + ACCOUNT_BALANCE.labels(str(variables.ACCOUNT.address)).set( + self.w3.eth.get_balance(variables.ACCOUNT.address) + ) + try: ( # Current frame's reference slot. @@ -404,12 +408,6 @@ def _get_latest_blockstamp(self) -> BlockStamp: logger.debug({'msg': 'Fetch latest blockstamp.', 'value': bs}) ORACLE_SLOT_NUMBER.labels('head').set(bs.slot_number) ORACLE_BLOCK_NUMBER.labels('head').set(bs.block_number) - - if variables.ACCOUNT: - ACCOUNT_BALANCE.labels(str(variables.ACCOUNT.address)).set( - self.w3.eth.get_balance(variables.ACCOUNT.address) - ) - return bs @lru_cache(maxsize=1) From 4a328c29426ce693e86907f8929b64aec1a61a56 Mon Sep 17 00:00:00 2001 From: hweawer Date: Fri, 24 Jan 2025 12:11:29 +0100 Subject: [PATCH 7/8] Add mocks for balance --- tests/modules/submodules/consensus/test_consensus.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index ada60f488..2b1cad25e 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -77,6 +77,7 @@ def test_get_latest_blockstamp(consensus, set_no_account): @pytest.mark.unit def test_get_member_info_with_account(consensus, set_report_account): bs = ReferenceBlockStampFactory.build() + consensus.w3.eth.get_balance = Mock(return_value=1) member_info = consensus.get_member_info(bs) assert isinstance(member_info, MemberInfo) @@ -90,6 +91,7 @@ def test_get_member_info_with_account(consensus, set_report_account): @pytest.mark.unit def test_get_member_info_without_account(consensus, set_no_account): bs = ReferenceBlockStampFactory.build() + consensus.w3.eth.get_balance = Mock(return_value=1) member_info = consensus.get_member_info(bs) assert isinstance(member_info, MemberInfo) @@ -103,6 +105,7 @@ def test_get_member_info_without_account(consensus, set_no_account): @pytest.mark.unit def test_get_member_info_no_member_account(consensus, set_not_member_account): bs = ReferenceBlockStampFactory.build() + consensus.w3.eth.get_balance = Mock(return_value=1) with pytest.raises(IsNotMemberException): consensus.get_member_info(bs) @@ -111,6 +114,7 @@ def test_get_member_info_no_member_account(consensus, set_not_member_account): @pytest.mark.unit def test_get_member_info_submit_only_account(consensus, set_submit_account): bs = ReferenceBlockStampFactory.build() + consensus.w3.eth.get_balance = Mock(return_value=1) member_info = consensus.get_member_info(bs) assert isinstance(member_info, MemberInfo) From 3ec3e9dc792c61805b939db512430e6ac8a14b37 Mon Sep 17 00:00:00 2001 From: hweawer Date: Fri, 24 Jan 2025 12:27:16 +0100 Subject: [PATCH 8/8] Fix test --- tests/modules/submodules/consensus/test_consensus.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index 2b1cad25e..f70f96790 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -173,6 +173,7 @@ def test_first_frame_is_not_yet_started(web3, consensus, caplog, use_account): consensus.get_chain_config = Mock(return_value=ChainConfigFactory.build()) first_frame = consensus.get_initial_or_current_frame(bs) + consensus.w3.eth.get_balance = Mock(return_value=1) member_info = consensus.get_member_info(bs) assert first_frame.ref_slot == 5 * 32 - 1