From 5afd959924b8a8efcd485bf9f3e51550b3e0e7c9 Mon Sep 17 00:00:00 2001 From: "kacper.golem" Date: Thu, 22 Feb 2024 13:41:25 +0100 Subject: [PATCH] convert eth to wei, exclude last block with reward, general refactor --- backend/app/context/epoch_details.py | 7 +++ backend/app/exceptions.py | 8 +++ .../external_api/bitquery/blocks_reward.py | 26 +++++----- .../external_api/bitquery/req_producer.py | 5 +- backend/app/legacy/utils/time.py | 8 +++ .../app/modules/common/timestamp_converter.py | 5 -- backend/app/modules/staking/proceeds/core.py | 13 ++++- .../staking/proceeds/service/aggregated.py | 49 ++++++++++--------- backend/tests/conftest.py | 13 ++--- backend/tests/helpers/context.py | 10 +++- .../bitquery/test_acc_blocks_reward.py | 15 ++++-- .../test_aggegated_staking_proceeds.py | 10 +++- 12 files changed, 111 insertions(+), 58 deletions(-) delete mode 100644 backend/app/modules/common/timestamp_converter.py diff --git a/backend/app/context/epoch_details.py b/backend/app/context/epoch_details.py index 47f75d69a2..b89fbb0e58 100644 --- a/backend/app/context/epoch_details.py +++ b/backend/app/context/epoch_details.py @@ -3,6 +3,7 @@ from app.context.epoch_state import EpochState from app.context.helpers import check_if_future +from app.exceptions import WrongBlocksRange from app.extensions import epochs from app.infrastructure import graphql from app.infrastructure.external_api.etherscan.blocks import get_block_num_from_ts @@ -50,6 +51,12 @@ def __init__( def duration_range(self) -> tuple[int, int]: return self.start_sec, self.end_sec + @property + def no_blocks(self): + if not self.end_block or not self.start_block: + raise WrongBlocksRange + return self.end_block - self.start_block + def get_epoch_details(epoch_num: int, epoch_state: EpochState) -> EpochDetails: if epoch_state == EpochState.FUTURE: diff --git a/backend/app/exceptions.py b/backend/app/exceptions.py index b9136d1891..677a8c1753 100644 --- a/backend/app/exceptions.py +++ b/backend/app/exceptions.py @@ -238,3 +238,11 @@ class EmptyAllocations(OctantException): def __init__(self): super().__init__(self.description, self.code) + + +class WrongBlocksRange(OctantException): + code = 400 + description = "Attempt to use wrong range of start and end block in epoch" + + def __init__(self): + super().__init__(self.description, self.code) diff --git a/backend/app/infrastructure/external_api/bitquery/blocks_reward.py b/backend/app/infrastructure/external_api/bitquery/blocks_reward.py index 49b38d1641..2d90af92b5 100644 --- a/backend/app/infrastructure/external_api/bitquery/blocks_reward.py +++ b/backend/app/infrastructure/external_api/bitquery/blocks_reward.py @@ -1,9 +1,9 @@ import requests + import app as app_module from app.constants import BITQUERY_API from app.exceptions import ExternalApiException -from app.extensions import w3 from app.infrastructure.external_api.bitquery.req_producer import ( produce_payload, BitQueryActions, @@ -11,20 +11,25 @@ ) -def accumulate_blocks_reward_wei(blocks: list) -> int: - blocks_reward_gwei = 0.0 - for block in blocks: - blocks_reward_gwei += float(block["reward"]) - - return int(w3.to_wei(blocks_reward_gwei, "gwei")) - +def get_blocks_rewards( + address: str, start_time: str, end_time: str, limit: int +) -> list: + """ + Fetch Ethereum blocks within a specified time range in ascending order by timestamp. -def get_blocks_reward(address: str, start_time: str, end_time: str) -> int: + Args: + - start_time (str): The start time in ISO 8601 format. + - end_time (str): The end time in ISO 8601 format. + - address (str): The miner (fee recipient) address. + - limit (int): The number of blocks to retrieve starting from start_time. + Useful whilst getting end_blocks exclusively from epochs. + """ payload = produce_payload( action_type=BitQueryActions.GET_BLOCK_REWARDS, address=address, start_time=start_time, end_time=end_time, + limit=limit, ) headers = get_bitquery_header() @@ -39,5 +44,4 @@ def get_blocks_reward(address: str, start_time: str, end_time: str) -> int: raise ExternalApiException(api_url, e, 500) blocks = json_response.json()["data"]["ethereum"]["blocks"] - blocks_reward = accumulate_blocks_reward_wei(blocks) - return blocks_reward + return blocks diff --git a/backend/app/infrastructure/external_api/bitquery/req_producer.py b/backend/app/infrastructure/external_api/bitquery/req_producer.py index 97f7b168e5..870eda3256 100644 --- a/backend/app/infrastructure/external_api/bitquery/req_producer.py +++ b/backend/app/infrastructure/external_api/bitquery/req_producer.py @@ -25,13 +25,13 @@ def produce_payload(action_type: BitQueryActions, **query_values) -> str: def _block_rewards_payload( - start_time: str, end_time: str, address: str, **kwargs + start_time: str, end_time: str, address: str, limit: int, **kwargs ) -> str: payload = json.dumps( { "query": f"""query ($network: EthereumNetwork!, $from: ISO8601DateTime, $till: ISO8601DateTime) {{ ethereum(network: $network) {{ - blocks(time: {{since: $from, till: $till}}) {{ + blocks(time: {{since: $from, till: $till}}, options: {{asc: "timestamp.unixtime", limit: {limit}}}) {{ timestamp {{ unixtime }} @@ -47,6 +47,7 @@ def _block_rewards_payload( "network": "ethereum", "from": start_time, "till": end_time, + "limit": limit, "dateFormat": "%Y-%m-%d", } ), diff --git a/backend/app/legacy/utils/time.py b/backend/app/legacy/utils/time.py index 15f4a64456..93d53a0640 100644 --- a/backend/app/legacy/utils/time.py +++ b/backend/app/legacy/utils/time.py @@ -19,6 +19,9 @@ def timestamp_s(self) -> float: def datetime(self) -> DateTime: return DateTime.fromtimestamp(self.timestamp_s()) + def to_isoformat(self): + return self.datetime().isoformat() + def __eq__(self, o): if isinstance(o, Timestamp): return self._timestamp_us == o._timestamp_us @@ -60,3 +63,8 @@ def sec_to_days(sec: int) -> int: def days_to_sec(days: int) -> int: return int(days * 86400) + + +def timestamp_to_isoformat(timestamp_sec: int) -> str: + timestamp = from_timestamp_s(timestamp_sec) + return timestamp.to_isoformat() diff --git a/backend/app/modules/common/timestamp_converter.py b/backend/app/modules/common/timestamp_converter.py deleted file mode 100644 index b3b7adcfdf..0000000000 --- a/backend/app/modules/common/timestamp_converter.py +++ /dev/null @@ -1,5 +0,0 @@ -from datetime import datetime - - -def timestamp_to_isoformat(timestamp_sec: int) -> str: - return datetime.fromtimestamp(timestamp_sec).isoformat() diff --git a/backend/app/modules/staking/proceeds/core.py b/backend/app/modules/staking/proceeds/core.py index ab141406d6..e156a0300a 100644 --- a/backend/app/modules/staking/proceeds/core.py +++ b/backend/app/modules/staking/proceeds/core.py @@ -1,3 +1,5 @@ +from decimal import Decimal + import pandas as pd from gmpy2 import mpz @@ -54,8 +56,15 @@ def sum_withdrawals(withdrawals_txs: list[dict]) -> int: return w3.to_wei(int(total_gwei), "gwei") -def aggregate_proceeds(mev: int, withdrawals: int, blocks_reward: int) -> int: - return mev + withdrawals + blocks_reward +def sum_blocks_rewards(blocks_rewards: list) -> int: + df = pd.DataFrame(blocks_rewards) + blocks_reward_eth = df["reward"].apply(Decimal).sum() + + return int(w3.to_wei(blocks_reward_eth, "ether")) + + +def aggregate_proceeds(mev: int, withdrawals: int, blocks_rewards: list) -> int: + return mev + withdrawals + sum_blocks_rewards(blocks_rewards) def _filter_deposit_withdrawals(amount: mpz) -> mpz: diff --git a/backend/app/modules/staking/proceeds/service/aggregated.py b/backend/app/modules/staking/proceeds/service/aggregated.py index 282f00d2de..ae121b87f4 100644 --- a/backend/app/modules/staking/proceeds/service/aggregated.py +++ b/backend/app/modules/staking/proceeds/service/aggregated.py @@ -5,8 +5,8 @@ get_transactions, AccountAction, ) -from app.infrastructure.external_api.bitquery.blocks_reward import get_blocks_reward -from app.modules.common.timestamp_converter import timestamp_to_isoformat +from app.infrastructure.external_api.bitquery.blocks_reward import get_blocks_rewards +from app.legacy.utils.time import timestamp_to_isoformat from app.modules.staking.proceeds.core import ( sum_mev, sum_withdrawals, @@ -16,23 +16,23 @@ class AggregatedStakingProceeds(Model): - def _compute_blocks_reward( - self, start_sec: int, end_sec: int, withdrawals_target: str - ) -> int: - blocks_reward = 0 + def _retrieve_blocks_rewards( + self, start_sec: int, end_sec: int, withdrawals_target: str, limit: int + ) -> list: + blocks_rewards = [] if end_sec is None: - return blocks_reward + return blocks_rewards start_datetime, end_datetime = ( timestamp_to_isoformat(start_sec), timestamp_to_isoformat(end_sec), ) - blocks_reward = get_blocks_reward( - withdrawals_target, start_datetime, end_datetime + blocks_rewards = get_blocks_rewards( + withdrawals_target, start_datetime, end_datetime, limit=limit ) - return blocks_reward + return blocks_rewards def get_staking_proceeds(self, context: Context) -> int: """ @@ -51,28 +51,33 @@ def get_staking_proceeds(self, context: Context) -> int: context.epoch_details.end_block, ) - if end_block is not None: - end_block -= 1 + start_sec, end_sec = context.epoch_details.duration_range + no_blocks_to_get = context.epoch_details.no_blocks + + blocks_rewards = self._retrieve_blocks_rewards( + start_sec, end_sec, withdrawals_target, limit=no_blocks_to_get + ) + end_block_for_transactions = end_block - 1 normal = get_transactions( - withdrawals_target, start_block, end_block, tx_type=AccountAction.NORMAL + withdrawals_target, + start_block, + end_block_for_transactions, + tx_type=AccountAction.NORMAL, ) internal = get_transactions( - withdrawals_target, start_block, end_block, tx_type=AccountAction.INTERNAL + withdrawals_target, + start_block, + end_block_for_transactions, + tx_type=AccountAction.INTERNAL, ) withdrawals = get_transactions( withdrawals_target, start_block, - end_block, + end_block_for_transactions, tx_type=AccountAction.BEACON_WITHDRAWAL, ) mev_value = sum_mev(withdrawals_target, normal, internal) withdrawals_value = sum_withdrawals(withdrawals) - start_sec, end_sec = context.epoch_details.duration_range - - blocks_reward = self._compute_blocks_reward( - start_sec, end_sec, withdrawals_target - ) - - return aggregate_proceeds(mev_value, withdrawals_value, blocks_reward) + return aggregate_proceeds(mev_value, withdrawals_value, blocks_rewards) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index eb928b44d0..a6c5eb68f6 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -22,9 +22,6 @@ from app.infrastructure.contracts.erc20 import ERC20 from app.infrastructure.contracts.proposals import Proposals from app.infrastructure.contracts.vault import Vault -from app.infrastructure.external_api.bitquery.blocks_reward import ( - accumulate_blocks_reward_wei, -) from app.legacy.controllers.allocations import allocate, deserialize_payload from app.legacy.core.allocations import Allocation, AllocationRequest from app.legacy.crypto.account import Account as CryptoAccount @@ -110,7 +107,7 @@ def mock_etherscan_api_get_block_num_from_ts(*args, **kwargs): return int(example_resp_json["result"]) -def mock_bitquery_api_get_blocks_reward(*args, **kwargs): +def mock_bitquery_api_get_blocks_rewards(*args, **kwargs): example_resp_json = { "data": { "ethereum": { @@ -135,7 +132,7 @@ def mock_bitquery_api_get_blocks_reward(*args, **kwargs): } blocks = example_resp_json["data"]["ethereum"]["blocks"] - return accumulate_blocks_reward_wei(blocks) + return blocks def pytest_addoption(parser): @@ -545,10 +542,10 @@ def patch_etherscan_get_block_api(monkeypatch): @pytest.fixture(scope="function") -def patch_bitquery_get_blocks_reward(monkeypatch): +def patch_bitquery_get_blocks_rewards(monkeypatch): monkeypatch.setattr( - "app.modules.staking.proceeds.service.aggregated.get_blocks_reward", - mock_bitquery_api_get_blocks_reward, + "app.modules.staking.proceeds.service.aggregated.get_blocks_rewards", + mock_bitquery_api_get_blocks_rewards, ) diff --git a/backend/tests/helpers/context.py b/backend/tests/helpers/context.py index 4b301104ff..92d36dc935 100644 --- a/backend/tests/helpers/context.py +++ b/backend/tests/helpers/context.py @@ -6,7 +6,13 @@ def get_epoch_details( - epoch_num: int, start=1000, duration=1000, decision_window=500, remaining_sec=1000 + epoch_num: int, + start=1000, + duration=1000, + decision_window=500, + remaining_sec=1000, + start_block: int = 12712551, + end_block: int = 12712551, ): return EpochDetails( epoch_num=epoch_num, @@ -14,6 +20,8 @@ def get_epoch_details( start=start, decision_window=decision_window, remaining_sec=remaining_sec, + start_block=start_block, + end_block=end_block, ) diff --git a/backend/tests/infrastracture/external_api/bitquery/test_acc_blocks_reward.py b/backend/tests/infrastracture/external_api/bitquery/test_acc_blocks_reward.py index b290d62b76..66bb7b4ef2 100644 --- a/backend/tests/infrastracture/external_api/bitquery/test_acc_blocks_reward.py +++ b/backend/tests/infrastracture/external_api/bitquery/test_acc_blocks_reward.py @@ -1,13 +1,18 @@ import pytest -from app.infrastructure.external_api.bitquery.blocks_reward import ( - accumulate_blocks_reward_wei, +from app.modules.staking.proceeds.core import ( + sum_blocks_rewards, ) @pytest.mark.parametrize( "blocks, result", - [([{"reward": 0.024473700594149782}, {"reward": 0.05342909432569912}], 77902794)], + [ + ( + [{"reward": 0.024473700594149782}, {"reward": 0.05342909432569912}], + 77902794919848899, + ) + ], ) -def test_accumulate_blocks_reward_wei(blocks, result): - actual_result = accumulate_blocks_reward_wei(blocks) +def test_sum_blocks_rewards(blocks, result): + actual_result = sum_blocks_rewards(blocks) assert actual_result == result diff --git a/backend/tests/modules/staking/test_aggegated_staking_proceeds.py b/backend/tests/modules/staking/test_aggegated_staking_proceeds.py index 1b1b7199ea..ec5c4396e6 100644 --- a/backend/tests/modules/staking/test_aggegated_staking_proceeds.py +++ b/backend/tests/modules/staking/test_aggegated_staking_proceeds.py @@ -9,11 +9,17 @@ def before(app): def test_aggregated_staking_proceeds( - patch_etherscan_transactions_api, patch_bitquery_get_blocks_reward + patch_etherscan_transactions_api, patch_bitquery_get_blocks_rewards ): + """ + Expected results for the test: + MEV 66813166811131780 + WITHDRAWALS 1498810000000000 + BLOCKS REWARD 77902794919848899 + """ context = get_context(1) service = AggregatedStakingProceeds() result = service.get_staking_proceeds(context) - assert result == 68311976_889034574 + assert result == 146214771_730980679