diff --git a/backend/app/constants.py b/backend/app/constants.py index 798d960572..daaec3f8aa 100644 --- a/backend/app/constants.py +++ b/backend/app/constants.py @@ -12,3 +12,4 @@ BEACONCHAIN_API = "https://beaconcha.in/api" ETHERSCAN_API = "https://api.etherscan.io/api" +BITQUERY_API = "https://graphql.bitquery.io" diff --git a/backend/app/context/epoch_details.py b/backend/app/context/epoch_details.py index ecbcd6bd95..0000140c40 100644 --- a/backend/app/context/epoch_details.py +++ b/backend/app/context/epoch_details.py @@ -1,8 +1,9 @@ from datetime import datetime -from typing import List +from typing import List, Tuple from app.context.epoch_state import EpochState from app.context.helpers import check_if_future +from app.exceptions import InvalidBlocksRange from app.extensions import epochs from app.infrastructure import graphql from app.infrastructure.external_api.etherscan.blocks import get_block_num_from_ts @@ -44,6 +45,20 @@ def __init__( self.remaining_sec = remaining_sec self.remaining_days = sec_to_days(self.remaining_sec) + self.block_rewards = None + + @property + def duration_range(self) -> Tuple[int, int]: + return self.start_sec, self.end_sec + + @property + def no_blocks(self): + """ + Returns the number of blocks within [start_block, end_block) in the epoch. + """ + if not self.end_block or not self.start_block: + raise InvalidBlocksRange + return self.end_block - self.start_block def get_epoch_details(epoch_num: int, epoch_state: EpochState) -> EpochDetails: diff --git a/backend/app/exceptions.py b/backend/app/exceptions.py index b9136d1891..de6de44331 100644 --- a/backend/app/exceptions.py +++ b/backend/app/exceptions.py @@ -238,3 +238,11 @@ class EmptyAllocations(OctantException): def __init__(self): super().__init__(self.description, self.code) + + +class InvalidBlocksRange(OctantException): + code = 400 + description = "Attempt to use wrong range of start and end block in epoch" + + def __init__(self): + super().__init__(self.description, self.code) diff --git a/backend/app/infrastructure/external_api/bitquery/__init__.py b/backend/app/infrastructure/external_api/bitquery/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/app/infrastructure/external_api/bitquery/blocks_reward.py b/backend/app/infrastructure/external_api/bitquery/blocks_reward.py new file mode 100644 index 0000000000..2d90af92b5 --- /dev/null +++ b/backend/app/infrastructure/external_api/bitquery/blocks_reward.py @@ -0,0 +1,47 @@ +import requests + + +import app as app_module +from app.constants import BITQUERY_API +from app.exceptions import ExternalApiException +from app.infrastructure.external_api.bitquery.req_producer import ( + produce_payload, + BitQueryActions, + get_bitquery_header, +) + + +def get_blocks_rewards( + address: str, start_time: str, end_time: str, limit: int +) -> list: + """ + Fetch Ethereum blocks within a specified time range in ascending order by timestamp. + + Args: + - start_time (str): The start time in ISO 8601 format. + - end_time (str): The end time in ISO 8601 format. + - address (str): The miner (fee recipient) address. + - limit (int): The number of blocks to retrieve starting from start_time. + Useful whilst getting end_blocks exclusively from epochs. + """ + payload = produce_payload( + action_type=BitQueryActions.GET_BLOCK_REWARDS, + address=address, + start_time=start_time, + end_time=end_time, + limit=limit, + ) + headers = get_bitquery_header() + + api_url = BITQUERY_API + + try: + response = requests.request("POST", api_url, headers=headers, data=payload) + response.raise_for_status() + json_response = response.json() + except requests.exceptions.RequestException as e: + app_module.ExceptionHandler.print_stacktrace(e) + raise ExternalApiException(api_url, e, 500) + + blocks = json_response.json()["data"]["ethereum"]["blocks"] + return blocks diff --git a/backend/app/infrastructure/external_api/bitquery/req_producer.py b/backend/app/infrastructure/external_api/bitquery/req_producer.py new file mode 100644 index 0000000000..870eda3256 --- /dev/null +++ b/backend/app/infrastructure/external_api/bitquery/req_producer.py @@ -0,0 +1,57 @@ +import json +from enum import IntEnum + +from flask import current_app as app + + +class BitQueryActions(IntEnum): + GET_BLOCK_REWARDS = 0 + + +def get_bitquery_header(): + headers = { + "Content-Type": "application/json", + "X-API-KEY": app.config["BITQUERY_API_KEY"], + "Authorization": app.config["BITQUERY_BEARER"], + } + + return headers + + +def produce_payload(action_type: BitQueryActions, **query_values) -> str: + payloads_variations = {BitQueryActions.GET_BLOCK_REWARDS: _block_rewards_payload} + + return payloads_variations[action_type](**query_values) + + +def _block_rewards_payload( + start_time: str, end_time: str, address: str, limit: int, **kwargs +) -> str: + payload = json.dumps( + { + "query": f"""query ($network: EthereumNetwork!, $from: ISO8601DateTime, $till: ISO8601DateTime) {{ + ethereum(network: $network) {{ + blocks(time: {{since: $from, till: $till}}, options: {{asc: "timestamp.unixtime", limit: {limit}}}) {{ + timestamp {{ + unixtime + }} + reward + address: miner(miner: {{is: "{address}"}}) {{ + address + }} + }} + }} + }}""", + "variables": json.dumps( + { + "network": "ethereum", + "from": start_time, + "till": end_time, + "limit": limit, + "dateFormat": "%Y-%m-%d", + } + ), + } + ) + + return payload diff --git a/backend/app/infrastructure/external_api/etherscan/blocks.py b/backend/app/infrastructure/external_api/etherscan/blocks.py index b770917192..558686bf73 100644 --- a/backend/app/infrastructure/external_api/etherscan/blocks.py +++ b/backend/app/infrastructure/external_api/etherscan/blocks.py @@ -12,7 +12,7 @@ def get_block_num_from_ts(timestamp: int) -> int: app.logger.debug(f"Getting block number from timestamp: {timestamp}") - api_url = _get_api_url(timestamp, BlockAction.BLOCK) + api_url = _get_api_url(timestamp, BlockAction.BLOCK_NO_BY_TS) try: response = requests.get(api_url) raise_for_status(response) diff --git a/backend/app/infrastructure/external_api/etherscan/req_params.py b/backend/app/infrastructure/external_api/etherscan/req_params.py index 3034b8fa3e..8402666624 100644 --- a/backend/app/infrastructure/external_api/etherscan/req_params.py +++ b/backend/app/infrastructure/external_api/etherscan/req_params.py @@ -8,7 +8,8 @@ class AccountAction(StrEnum): class BlockAction(StrEnum): - BLOCK = "getblocknobytime" + BLOCK_NO_BY_TS = "getblocknobytime" + BLOCK_REWARD = "getblockreward" class ClosestValue(StrEnum): diff --git a/backend/app/legacy/utils/time.py b/backend/app/legacy/utils/time.py index 15f4a64456..93d53a0640 100644 --- a/backend/app/legacy/utils/time.py +++ b/backend/app/legacy/utils/time.py @@ -19,6 +19,9 @@ def timestamp_s(self) -> float: def datetime(self) -> DateTime: return DateTime.fromtimestamp(self.timestamp_s()) + def to_isoformat(self): + return self.datetime().isoformat() + def __eq__(self, o): if isinstance(o, Timestamp): return self._timestamp_us == o._timestamp_us @@ -60,3 +63,8 @@ def sec_to_days(sec: int) -> int: def days_to_sec(days: int) -> int: return int(days * 86400) + + +def timestamp_to_isoformat(timestamp_sec: int) -> str: + timestamp = from_timestamp_s(timestamp_sec) + return timestamp.to_isoformat() diff --git a/backend/app/modules/staking/proceeds/core.py b/backend/app/modules/staking/proceeds/core.py index 1fd4085c74..e156a0300a 100644 --- a/backend/app/modules/staking/proceeds/core.py +++ b/backend/app/modules/staking/proceeds/core.py @@ -1,3 +1,5 @@ +from decimal import Decimal + import pandas as pd from gmpy2 import mpz @@ -54,8 +56,15 @@ def sum_withdrawals(withdrawals_txs: list[dict]) -> int: return w3.to_wei(int(total_gwei), "gwei") -def aggregate_proceeds(mev: int, withdrawals: int) -> int: - return mev + withdrawals +def sum_blocks_rewards(blocks_rewards: list) -> int: + df = pd.DataFrame(blocks_rewards) + blocks_reward_eth = df["reward"].apply(Decimal).sum() + + return int(w3.to_wei(blocks_reward_eth, "ether")) + + +def aggregate_proceeds(mev: int, withdrawals: int, blocks_rewards: list) -> int: + return mev + withdrawals + sum_blocks_rewards(blocks_rewards) def _filter_deposit_withdrawals(amount: mpz) -> mpz: diff --git a/backend/app/modules/staking/proceeds/service/aggregated.py b/backend/app/modules/staking/proceeds/service/aggregated.py index 7b827386a2..ae121b87f4 100644 --- a/backend/app/modules/staking/proceeds/service/aggregated.py +++ b/backend/app/modules/staking/proceeds/service/aggregated.py @@ -5,6 +5,8 @@ get_transactions, AccountAction, ) +from app.infrastructure.external_api.bitquery.blocks_reward import get_blocks_rewards +from app.legacy.utils.time import timestamp_to_isoformat from app.modules.staking.proceeds.core import ( sum_mev, sum_withdrawals, @@ -14,6 +16,24 @@ class AggregatedStakingProceeds(Model): + def _retrieve_blocks_rewards( + self, start_sec: int, end_sec: int, withdrawals_target: str, limit: int + ) -> list: + blocks_rewards = [] + + if end_sec is None: + return blocks_rewards + + start_datetime, end_datetime = ( + timestamp_to_isoformat(start_sec), + timestamp_to_isoformat(end_sec), + ) + + blocks_rewards = get_blocks_rewards( + withdrawals_target, start_datetime, end_datetime, limit=limit + ) + return blocks_rewards + def get_staking_proceeds(self, context: Context) -> int: """ Retrieves a list of transactions, calculates MEV value and aggregates it with withdrawals. @@ -31,23 +51,33 @@ def get_staking_proceeds(self, context: Context) -> int: context.epoch_details.end_block, ) - if end_block is not None: - end_block -= 1 + start_sec, end_sec = context.epoch_details.duration_range + no_blocks_to_get = context.epoch_details.no_blocks + blocks_rewards = self._retrieve_blocks_rewards( + start_sec, end_sec, withdrawals_target, limit=no_blocks_to_get + ) + + end_block_for_transactions = end_block - 1 normal = get_transactions( - withdrawals_target, start_block, end_block, tx_type=AccountAction.NORMAL + withdrawals_target, + start_block, + end_block_for_transactions, + tx_type=AccountAction.NORMAL, ) internal = get_transactions( - withdrawals_target, start_block, end_block, tx_type=AccountAction.INTERNAL + withdrawals_target, + start_block, + end_block_for_transactions, + tx_type=AccountAction.INTERNAL, ) withdrawals = get_transactions( withdrawals_target, start_block, - end_block, + end_block_for_transactions, tx_type=AccountAction.BEACON_WITHDRAWAL, ) - mev_value = sum_mev(withdrawals_target, normal, internal) withdrawals_value = sum_withdrawals(withdrawals) - return aggregate_proceeds(mev_value, withdrawals_value) + return aggregate_proceeds(mev_value, withdrawals_value, blocks_rewards) diff --git a/backend/app/settings.py b/backend/app/settings.py index 958a6e5758..b36313eb14 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -23,6 +23,8 @@ class Config(object): SUBGRAPH_ENDPOINT = os.getenv("SUBGRAPH_ENDPOINT") WEB3_PROVIDER = Web3.HTTPProvider(os.getenv("ETH_RPC_PROVIDER_URL")) ETHERSCAN_API_KEY = os.getenv("ETHERSCAN_API_KEY") + BITQUERY_API_KEY = os.getenv("BITQUERY_API_KEY") + BITQUERY_BEARER = os.getenv("BITQUERY_BEARER") SCHEDULER_ENABLED = _parse_bool(os.getenv("SCHEDULER_ENABLED")) CACHE_TYPE = "SimpleCache" diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 8a1f7efb4f..a6c5eb68f6 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -9,6 +9,11 @@ import gql import pytest +from eth_account import Account +from flask import g as request_context +from flask.testing import FlaskClient +from web3 import Web3 + from app import create_app from app.engine.user.effective_deposit import DepositEvent, EventType, UserDeposit from app.extensions import db, deposits, glm, gql_factory, w3 @@ -23,9 +28,6 @@ from app.legacy.crypto.eip712 import build_allocations_eip712_data, sign from app.modules.dto import AccountFundsDTO from app.settings import DevConfig, TestConfig -from eth_account import Account -from flask import g as request_context -from flask.testing import FlaskClient from tests.helpers.constants import ( ALICE, ALL_INDIVIDUAL_REWARDS, @@ -58,7 +60,6 @@ from tests.helpers.mocked_epoch_details import EPOCH_EVENTS from tests.helpers.octant_rewards import octant_rewards from tests.helpers.subgraph.events import create_deposit_event -from web3 import Web3 # Contracts mocks MOCK_EPOCHS = MagicMock(spec=Epochs) @@ -106,6 +107,34 @@ def mock_etherscan_api_get_block_num_from_ts(*args, **kwargs): return int(example_resp_json["result"]) +def mock_bitquery_api_get_blocks_rewards(*args, **kwargs): + example_resp_json = { + "data": { + "ethereum": { + "blocks": [ + { + "timestamp": {"unixtime": 1708448963}, + "reward": 0.024473700594149782, + "address": { + "address": "0x1f9090aae28b8a3dceadf281b0f12828e676c326" + }, + }, + { + "timestamp": {"unixtime": 1708449035}, + "reward": 0.05342909432569912, + "address": { + "address": "0x1f9090aae28b8a3dceadf281b0f12828e676c326" + }, + }, + ] + } + } + } + + blocks = example_resp_json["data"]["ethereum"]["blocks"] + return blocks + + def pytest_addoption(parser): parser.addoption( "--runapi", @@ -512,6 +541,14 @@ def patch_etherscan_get_block_api(monkeypatch): ) +@pytest.fixture(scope="function") +def patch_bitquery_get_blocks_rewards(monkeypatch): + monkeypatch.setattr( + "app.modules.staking.proceeds.service.aggregated.get_blocks_rewards", + mock_bitquery_api_get_blocks_rewards, + ) + + @pytest.fixture(scope="function") def mock_users_db(app, user_accounts): alice = database.user.add_user(user_accounts[0].address) diff --git a/backend/tests/helpers/context.py b/backend/tests/helpers/context.py index 4b301104ff..92d36dc935 100644 --- a/backend/tests/helpers/context.py +++ b/backend/tests/helpers/context.py @@ -6,7 +6,13 @@ def get_epoch_details( - epoch_num: int, start=1000, duration=1000, decision_window=500, remaining_sec=1000 + epoch_num: int, + start=1000, + duration=1000, + decision_window=500, + remaining_sec=1000, + start_block: int = 12712551, + end_block: int = 12712551, ): return EpochDetails( epoch_num=epoch_num, @@ -14,6 +20,8 @@ def get_epoch_details( start=start, decision_window=decision_window, remaining_sec=remaining_sec, + start_block=start_block, + end_block=end_block, ) diff --git a/backend/tests/infrastracture/external_api/bitquery/test_acc_blocks_reward.py b/backend/tests/infrastracture/external_api/bitquery/test_acc_blocks_reward.py new file mode 100644 index 0000000000..66bb7b4ef2 --- /dev/null +++ b/backend/tests/infrastracture/external_api/bitquery/test_acc_blocks_reward.py @@ -0,0 +1,18 @@ +import pytest +from app.modules.staking.proceeds.core import ( + sum_blocks_rewards, +) + + +@pytest.mark.parametrize( + "blocks, result", + [ + ( + [{"reward": 0.024473700594149782}, {"reward": 0.05342909432569912}], + 77902794919848899, + ) + ], +) +def test_sum_blocks_rewards(blocks, result): + actual_result = sum_blocks_rewards(blocks) + assert actual_result == result diff --git a/backend/tests/modules/staking/test_aggegated_staking_proceeds.py b/backend/tests/modules/staking/test_aggegated_staking_proceeds.py index b77071052e..ec5c4396e6 100644 --- a/backend/tests/modules/staking/test_aggegated_staking_proceeds.py +++ b/backend/tests/modules/staking/test_aggegated_staking_proceeds.py @@ -8,10 +8,18 @@ def before(app): pass -def test_aggregated_staking_proceeds(patch_etherscan_transactions_api): +def test_aggregated_staking_proceeds( + patch_etherscan_transactions_api, patch_bitquery_get_blocks_rewards +): + """ + Expected results for the test: + MEV 66813166811131780 + WITHDRAWALS 1498810000000000 + BLOCKS REWARD 77902794919848899 + """ context = get_context(1) service = AggregatedStakingProceeds() result = service.get_staking_proceeds(context) - assert result == 68311976_811131780 + assert result == 146214771_730980679