From 07f7dd92b522f964a1bd7d934ce5f65926daaf2c Mon Sep 17 00:00:00 2001 From: "kacper.golem" Date: Tue, 20 Feb 2024 21:22:22 +0100 Subject: [PATCH] improve api communication with bitquery --- backend/app/constants.py | 1 + backend/app/context/epoch_details.py | 5 ++ .../external_api/bitquery/__init__.py | 0 .../external_api/bitquery/blocks_reward.py | 43 ++++++++++++++ .../external_api/bitquery/req_producer.py | 56 +++++++++++++++++++ .../external_api/etherscan/blocks_reward.py | 44 --------------- .../app/modules/common/timestamp_converter.py | 5 ++ .../staking/proceeds/service/aggregated.py | 21 ++++--- backend/app/settings.py | 1 + backend/tests/conftest.py | 48 ++++++++++++++-- .../test_aggegated_staking_proceeds.py | 6 +- 11 files changed, 173 insertions(+), 57 deletions(-) create mode 100644 backend/app/infrastructure/external_api/bitquery/__init__.py create mode 100644 backend/app/infrastructure/external_api/bitquery/blocks_reward.py create mode 100644 backend/app/infrastructure/external_api/bitquery/req_producer.py delete mode 100644 backend/app/infrastructure/external_api/etherscan/blocks_reward.py create mode 100644 backend/app/modules/common/timestamp_converter.py diff --git a/backend/app/constants.py b/backend/app/constants.py index 798d960572..daaec3f8aa 100644 --- a/backend/app/constants.py +++ b/backend/app/constants.py @@ -12,3 +12,4 @@ BEACONCHAIN_API = "https://beaconcha.in/api" ETHERSCAN_API = "https://api.etherscan.io/api" +BITQUERY_API = "https://graphql.bitquery.io" diff --git a/backend/app/context/epoch_details.py b/backend/app/context/epoch_details.py index ecbcd6bd95..47f75d69a2 100644 --- a/backend/app/context/epoch_details.py +++ b/backend/app/context/epoch_details.py @@ -44,6 +44,11 @@ def __init__( self.remaining_sec = remaining_sec self.remaining_days = sec_to_days(self.remaining_sec) + self.block_rewards = None + + @property + def duration_range(self) -> tuple[int, int]: + return self.start_sec, self.end_sec def get_epoch_details(epoch_num: int, epoch_state: EpochState) -> EpochDetails: diff --git a/backend/app/infrastructure/external_api/bitquery/__init__.py b/backend/app/infrastructure/external_api/bitquery/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/app/infrastructure/external_api/bitquery/blocks_reward.py b/backend/app/infrastructure/external_api/bitquery/blocks_reward.py new file mode 100644 index 0000000000..505f5136cc --- /dev/null +++ b/backend/app/infrastructure/external_api/bitquery/blocks_reward.py @@ -0,0 +1,43 @@ +import requests + +import app as app_module +from app.constants import BITQUERY_API +from app.exceptions import ExternalApiException +from app.extensions import w3 +from app.infrastructure.external_api.bitquery.req_producer import ( + produce_payload, + BitQueryActions, + get_bitquery_header, +) + + +def accumulate_blocks_reward(blocks: list) -> int: + blocks_reward_gwei = 0.0 + for block in blocks: + blocks_reward_gwei += float(block["reward"]) + + return int(w3.to_wei(blocks_reward_gwei, "gwei")) + + +def get_blocks_reward(address: str, start_time: str, end_time: str) -> int: + payload = produce_payload( + action_type=BitQueryActions.GET_BLOCK_REWARDS, + address=address, + start_time=start_time, + end_time=end_time, + ) + headers = get_bitquery_header() + + api_url = BITQUERY_API + + try: + response = requests.request("POST", api_url, headers=headers, data=payload) + response.raise_for_status() + json_response = response.json() + except requests.exceptions.RequestException as e: + app_module.ExceptionHandler.print_stacktrace(e) + raise ExternalApiException(api_url, e, 500) + + blocks = json_response.json()["data"]["ethereum"]["blocks"] + blocks_reward = accumulate_blocks_reward(blocks) + return blocks_reward diff --git a/backend/app/infrastructure/external_api/bitquery/req_producer.py b/backend/app/infrastructure/external_api/bitquery/req_producer.py new file mode 100644 index 0000000000..ea24c685c7 --- /dev/null +++ b/backend/app/infrastructure/external_api/bitquery/req_producer.py @@ -0,0 +1,56 @@ +import json +from enum import IntEnum + +from flask import current_app as app + + +class BitQueryActions(IntEnum): + GET_BLOCK_REWARDS = 0 + + +def get_bitquery_header(): + headers = { + "Content-Type": "application/json", + "X-API-KEY": app.config["BITQUERY_API_KEY"], + "Authorization": "Bearer ory_at_yeIAwzWgtkssnWzCCpDKnrDv74AEkhaRbO1VHsrCSz0.nVnX2zUQiYr2AKSu823GYOuyIsiyVyp5WLDBFcawQT8", + } + + return headers + + +def produce_payload(action_type: BitQueryActions, **query_values) -> str: + payloads_variations = {BitQueryActions.GET_BLOCK_REWARDS: _block_rewards_payload} + + return payloads_variations[action_type](**query_values) + + +def _block_rewards_payload( + start_time: str, end_time: str, address: str, **kwargs +) -> str: + payload = json.dumps( + { + "query": f"""query ($network: EthereumNetwork!, $from: ISO8601DateTime, $till: ISO8601DateTime) {{ + ethereum(network: $network) {{ + blocks(time: {{since: $from, till: $till}}) {{ + timestamp {{ + unixtime + }} + reward + address: miner(miner: {{is: "{address}"}}) {{ + address + }} + }} + }} + }}""", + "variables": json.dumps( + { + "network": "ethereum", + "from": start_time, + "till": end_time, + "dateFormat": "%Y-%m-%d", + } + ), + } + ) + + return payload diff --git a/backend/app/infrastructure/external_api/etherscan/blocks_reward.py b/backend/app/infrastructure/external_api/etherscan/blocks_reward.py deleted file mode 100644 index 49eebeedf3..0000000000 --- a/backend/app/infrastructure/external_api/etherscan/blocks_reward.py +++ /dev/null @@ -1,44 +0,0 @@ -import app as app_module -import requests -from app.constants import ETHERSCAN_API -from app.exceptions import ExternalApiException -from app.infrastructure.external_api.etherscan.helpers import raise_for_status -from app.infrastructure.external_api.etherscan.req_params import BlockAction -from flask import current_app as app - - -def get_blocks_reward(address: str, start_block: int, end_block: int) -> int: - app.logger.debug( - f"Getting blocks reward from {start_block} and {end_block} for {address} address" - ) - - block_reward = 0 - for i in range(start_block, end_block + 1): - api_url = _get_api_url(i, BlockAction.BLOCK_REWARD) - - try: - response = requests.get(api_url) - raise_for_status(response) - json_response = response.json() - except requests.exceptions.RequestException as e: - app_module.ExceptionHandler.print_stacktrace(e) - raise ExternalApiException(api_url, e, 500) - - result = json_response["result"] - if result["blockMiner"] == address: - block_reward += float(result["blockReward"]) - - return block_reward - - -def _get_api_url( - block_nr: int, - block_action: BlockAction, -) -> str: - api_key = app.config["ETHERSCAN_API_KEY"] - return ( - f"{ETHERSCAN_API}?module=block" - f"&action={block_action.value}" - f"&blockno={block_nr}" - f"&apikey={api_key}" - ) diff --git a/backend/app/modules/common/timestamp_converter.py b/backend/app/modules/common/timestamp_converter.py new file mode 100644 index 0000000000..b3b7adcfdf --- /dev/null +++ b/backend/app/modules/common/timestamp_converter.py @@ -0,0 +1,5 @@ +from datetime import datetime + + +def timestamp_to_isoformat(timestamp_sec: int) -> str: + return datetime.fromtimestamp(timestamp_sec).isoformat() diff --git a/backend/app/modules/staking/proceeds/service/aggregated.py b/backend/app/modules/staking/proceeds/service/aggregated.py index 03940e3cf7..4855b30cbf 100644 --- a/backend/app/modules/staking/proceeds/service/aggregated.py +++ b/backend/app/modules/staking/proceeds/service/aggregated.py @@ -5,7 +5,8 @@ get_transactions, AccountAction, ) -from app.infrastructure.external_api.etherscan.blocks_reward import get_blocks_reward +from app.infrastructure.external_api.bitquery.blocks_reward import get_blocks_reward +from app.modules.common.timestamp_converter import timestamp_to_isoformat from app.modules.staking.proceeds.core import ( sum_mev, sum_withdrawals, @@ -27,8 +28,6 @@ def get_staking_proceeds(self, context: Context) -> int: - int: Aggregated value for MEV and withdrawals. """ withdrawals_target = app.config["WITHDRAWALS_TARGET_CONTRACT_ADDRESS"].lower() - blocks_reward = 0 - start_block, end_block = ( context.epoch_details.start_block, context.epoch_details.end_block, @@ -36,9 +35,6 @@ def get_staking_proceeds(self, context: Context) -> int: if end_block is not None: end_block -= 1 - blocks_reward = get_blocks_reward( - withdrawals_target, start_block, end_block - ) normal = get_transactions( withdrawals_target, start_block, end_block, tx_type=AccountAction.NORMAL @@ -52,8 +48,19 @@ def get_staking_proceeds(self, context: Context) -> int: end_block, tx_type=AccountAction.BEACON_WITHDRAWAL, ) - mev_value = sum_mev(withdrawals_target, normal, internal) withdrawals_value = sum_withdrawals(withdrawals) + start_sec, end_sec = context.epoch_details.duration_range + blocks_reward = 0 + if end_sec is not None: + start_datetime, end_datetime = ( + timestamp_to_isoformat(start_sec), + timestamp_to_isoformat(end_sec), + ) + + blocks_reward = get_blocks_reward( + withdrawals_target, start_datetime, end_datetime + ) + return aggregate_proceeds(mev_value, withdrawals_value, blocks_reward) diff --git a/backend/app/settings.py b/backend/app/settings.py index 958a6e5758..57b269aa2e 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -23,6 +23,7 @@ class Config(object): SUBGRAPH_ENDPOINT = os.getenv("SUBGRAPH_ENDPOINT") WEB3_PROVIDER = Web3.HTTPProvider(os.getenv("ETH_RPC_PROVIDER_URL")) ETHERSCAN_API_KEY = os.getenv("ETHERSCAN_API_KEY") + BITQUERY_API_KEY = os.getenv("BITQUERY_API_KEY") SCHEDULER_ENABLED = _parse_bool(os.getenv("SCHEDULER_ENABLED")) CACHE_TYPE = "SimpleCache" diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 8a1f7efb4f..4fa3dfaf18 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -9,6 +9,11 @@ import gql import pytest +from eth_account import Account +from flask import g as request_context +from flask.testing import FlaskClient +from web3 import Web3 + from app import create_app from app.engine.user.effective_deposit import DepositEvent, EventType, UserDeposit from app.extensions import db, deposits, glm, gql_factory, w3 @@ -17,15 +22,15 @@ from app.infrastructure.contracts.erc20 import ERC20 from app.infrastructure.contracts.proposals import Proposals from app.infrastructure.contracts.vault import Vault +from app.infrastructure.external_api.bitquery.blocks_reward import ( + accumulate_blocks_reward, +) from app.legacy.controllers.allocations import allocate, deserialize_payload from app.legacy.core.allocations import Allocation, AllocationRequest from app.legacy.crypto.account import Account as CryptoAccount from app.legacy.crypto.eip712 import build_allocations_eip712_data, sign from app.modules.dto import AccountFundsDTO from app.settings import DevConfig, TestConfig -from eth_account import Account -from flask import g as request_context -from flask.testing import FlaskClient from tests.helpers.constants import ( ALICE, ALL_INDIVIDUAL_REWARDS, @@ -58,7 +63,6 @@ from tests.helpers.mocked_epoch_details import EPOCH_EVENTS from tests.helpers.octant_rewards import octant_rewards from tests.helpers.subgraph.events import create_deposit_event -from web3 import Web3 # Contracts mocks MOCK_EPOCHS = MagicMock(spec=Epochs) @@ -106,6 +110,34 @@ def mock_etherscan_api_get_block_num_from_ts(*args, **kwargs): return int(example_resp_json["result"]) +def mock_bitquery_api_get_blocks_reward(*args, **kwargs): + example_resp_json = { + "data": { + "ethereum": { + "blocks": [ + { + "timestamp": {"unixtime": 1708448963}, + "reward": 0.024473700594149782, + "address": { + "address": "0x1f9090aae28b8a3dceadf281b0f12828e676c326" + }, + }, + { + "timestamp": {"unixtime": 1708449035}, + "reward": 0.05342909432569912, + "address": { + "address": "0x1f9090aae28b8a3dceadf281b0f12828e676c326" + }, + }, + ] + } + } + } + + blocks = example_resp_json["data"]["ethereum"]["blocks"] + return accumulate_blocks_reward(blocks) + + def pytest_addoption(parser): parser.addoption( "--runapi", @@ -512,6 +544,14 @@ def patch_etherscan_get_block_api(monkeypatch): ) +@pytest.fixture(scope="function") +def patch_bitquery_get_blocks_reward(monkeypatch): + monkeypatch.setattr( + "app.modules.staking.proceeds.service.aggregated.get_blocks_reward", + mock_bitquery_api_get_blocks_reward, + ) + + @pytest.fixture(scope="function") def mock_users_db(app, user_accounts): alice = database.user.add_user(user_accounts[0].address) diff --git a/backend/tests/modules/staking/test_aggegated_staking_proceeds.py b/backend/tests/modules/staking/test_aggegated_staking_proceeds.py index b77071052e..1b1b7199ea 100644 --- a/backend/tests/modules/staking/test_aggegated_staking_proceeds.py +++ b/backend/tests/modules/staking/test_aggegated_staking_proceeds.py @@ -8,10 +8,12 @@ def before(app): pass -def test_aggregated_staking_proceeds(patch_etherscan_transactions_api): +def test_aggregated_staking_proceeds( + patch_etherscan_transactions_api, patch_bitquery_get_blocks_reward +): context = get_context(1) service = AggregatedStakingProceeds() result = service.get_staking_proceeds(context) - assert result == 68311976_811131780 + assert result == 68311976_889034574