improve api communication with bitquery

backend/app/constants.py

@@ -12,3 +12,4 @@
 
 BEACONCHAIN_API = "https://beaconcha.in/api"
 ETHERSCAN_API = "https://api.etherscan.io/api"
+BITQUERY_API = "https://graphql.bitquery.io"

backend/app/context/epoch_details.py

@@ -44,6 +44,11 @@ def __init__(
             self.remaining_sec = remaining_sec
 
         self.remaining_days = sec_to_days(self.remaining_sec)
+        self.block_rewards = None
+
+    @property
+    def duration_range(self) -> tuple[int, int]:
+        return self.start_sec, self.end_sec
 
 
 def get_epoch_details(epoch_num: int, epoch_state: EpochState) -> EpochDetails:

backend/app/infrastructure/external_api/bitquery/blocks_reward.py

@@ -0,0 +1,43 @@
+import requests
+
+import app as app_module
+from app.constants import BITQUERY_API
+from app.exceptions import ExternalApiException
+from app.extensions import w3
+from app.infrastructure.external_api.bitquery.req_producer import (
+    produce_payload,
+    BitQueryActions,
+    get_bitquery_header,
+)
+
+
+def accumulate_blocks_reward(blocks: list) -> int:
+    blocks_reward_gwei = 0.0
+    for block in blocks:
+        blocks_reward_gwei += float(block["reward"])
+
+    return int(w3.to_wei(blocks_reward_gwei, "gwei"))
+
+
+def get_blocks_reward(address: str, start_time: str, end_time: str) -> int:
+    payload = produce_payload(
+        action_type=BitQueryActions.GET_BLOCK_REWARDS,
+        address=address,
+        start_time=start_time,
+        end_time=end_time,
+    )
+    headers = get_bitquery_header()
+
+    api_url = BITQUERY_API
+
+    try:
+        response = requests.request("POST", api_url, headers=headers, data=payload)
+        response.raise_for_status()
+        json_response = response.json()
+    except requests.exceptions.RequestException as e:
+        app_module.ExceptionHandler.print_stacktrace(e)
+        raise ExternalApiException(api_url, e, 500)
+
+    blocks = json_response.json()["data"]["ethereum"]["blocks"]
+    blocks_reward = accumulate_blocks_reward(blocks)
+    return blocks_reward

backend/app/infrastructure/external_api/bitquery/req_producer.py

@@ -0,0 +1,56 @@
+import json
+from enum import IntEnum
+
+from flask import current_app as app
+
+
+class BitQueryActions(IntEnum):
+    GET_BLOCK_REWARDS = 0
+
+
+def get_bitquery_header():
+    headers = {
+        "Content-Type": "application/json",
+        "X-API-KEY": app.config["BITQUERY_API_KEY"],
+        "Authorization": "Bearer ory_at_yeIAwzWgtkssnWzCCpDKnrDv74AEkhaRbO1VHsrCSz0.nVnX2zUQiYr2AKSu823GYOuyIsiyVyp5WLDBFcawQT8",
+    }
+
+    return headers
+
+
+def produce_payload(action_type: BitQueryActions, **query_values) -> str:
+    payloads_variations = {BitQueryActions.GET_BLOCK_REWARDS: _block_rewards_payload}
+
+    return payloads_variations[action_type](**query_values)
+
+
+def _block_rewards_payload(
+    start_time: str, end_time: str, address: str, **kwargs
+) -> str:
+    payload = json.dumps(
+        {
+            "query": f"""query ($network: EthereumNetwork!, $from: ISO8601DateTime, $till: ISO8601DateTime) {{
+          ethereum(network: $network) {{
+            blocks(time: {{since: $from, till: $till}}) {{
+              timestamp {{
+                unixtime
+              }}
+              reward
+              address: miner(miner: {{is: "{address}"}}) {{
+                address
+              }}
+            }}
+          }}
+        }}""",
+            "variables": json.dumps(
+                {
+                    "network": "ethereum",
+                    "from": start_time,
+                    "till": end_time,
+                    "dateFormat": "%Y-%m-%d",
+                }
+            ),
+        }
+    )
+
+    return payload

backend/app/modules/common/timestamp_converter.py

@@ -0,0 +1,5 @@
+from datetime import datetime
+
+
+def timestamp_to_isoformat(timestamp_sec: int) -> str:
+    return datetime.fromtimestamp(timestamp_sec).isoformat()

backend/app/modules/staking/proceeds/service/aggregated.py

@@ -5,7 +5,8 @@
     get_transactions,
     AccountAction,
 )
-from app.infrastructure.external_api.etherscan.blocks_reward import get_blocks_reward
+from app.infrastructure.external_api.bitquery.blocks_reward import get_blocks_reward
+from app.modules.common.timestamp_converter import timestamp_to_isoformat
 from app.modules.staking.proceeds.core import (
     sum_mev,
     sum_withdrawals,
@@ -27,18 +28,13 @@ def get_staking_proceeds(self, context: Context) -> int:
         - int: Aggregated value for MEV and withdrawals.
         """
         withdrawals_target = app.config["WITHDRAWALS_TARGET_CONTRACT_ADDRESS"].lower()
-        blocks_reward = 0
-
         start_block, end_block = (
             context.epoch_details.start_block,
             context.epoch_details.end_block,
         )
 
         if end_block is not None:
             end_block -= 1
-            blocks_reward = get_blocks_reward(
-                withdrawals_target, start_block, end_block
-            )
 
         normal = get_transactions(
             withdrawals_target, start_block, end_block, tx_type=AccountAction.NORMAL
@@ -52,8 +48,19 @@ def get_staking_proceeds(self, context: Context) -> int:
             end_block,
             tx_type=AccountAction.BEACON_WITHDRAWAL,
         )
-
         mev_value = sum_mev(withdrawals_target, normal, internal)
         withdrawals_value = sum_withdrawals(withdrawals)
 
+        start_sec, end_sec = context.epoch_details.duration_range
+        blocks_reward = 0
+        if end_sec is not None:
+            start_datetime, end_datetime = (
+                timestamp_to_isoformat(start_sec),
+                timestamp_to_isoformat(end_sec),
+            )
+
+            blocks_reward = get_blocks_reward(
+                withdrawals_target, start_datetime, end_datetime
+            )
+
         return aggregate_proceeds(mev_value, withdrawals_value, blocks_reward)

backend/app/settings.py

@@ -23,6 +23,7 @@ class Config(object):
     SUBGRAPH_ENDPOINT = os.getenv("SUBGRAPH_ENDPOINT")
     WEB3_PROVIDER = Web3.HTTPProvider(os.getenv("ETH_RPC_PROVIDER_URL"))
     ETHERSCAN_API_KEY = os.getenv("ETHERSCAN_API_KEY")
+    BITQUERY_API_KEY = os.getenv("BITQUERY_API_KEY")
     SCHEDULER_ENABLED = _parse_bool(os.getenv("SCHEDULER_ENABLED"))
     CACHE_TYPE = "SimpleCache"
 

backend/tests/conftest.py

@@ -9,6 +9,11 @@
 
 import gql
 import pytest
+from eth_account import Account
+from flask import g as request_context
+from flask.testing import FlaskClient
+from web3 import Web3
+
 from app import create_app
 from app.engine.user.effective_deposit import DepositEvent, EventType, UserDeposit
 from app.extensions import db, deposits, glm, gql_factory, w3
@@ -17,15 +22,15 @@
 from app.infrastructure.contracts.erc20 import ERC20
 from app.infrastructure.contracts.proposals import Proposals
 from app.infrastructure.contracts.vault import Vault
+from app.infrastructure.external_api.bitquery.blocks_reward import (
+    accumulate_blocks_reward,
+)
 from app.legacy.controllers.allocations import allocate, deserialize_payload
 from app.legacy.core.allocations import Allocation, AllocationRequest
 from app.legacy.crypto.account import Account as CryptoAccount
 from app.legacy.crypto.eip712 import build_allocations_eip712_data, sign
 from app.modules.dto import AccountFundsDTO
 from app.settings import DevConfig, TestConfig
-from eth_account import Account
-from flask import g as request_context
-from flask.testing import FlaskClient
 from tests.helpers.constants import (
     ALICE,
     ALL_INDIVIDUAL_REWARDS,
@@ -58,7 +63,6 @@
 from tests.helpers.mocked_epoch_details import EPOCH_EVENTS
 from tests.helpers.octant_rewards import octant_rewards
 from tests.helpers.subgraph.events import create_deposit_event
-from web3 import Web3
 
 # Contracts mocks
 MOCK_EPOCHS = MagicMock(spec=Epochs)
@@ -106,6 +110,34 @@ def mock_etherscan_api_get_block_num_from_ts(*args, **kwargs):
     return int(example_resp_json["result"])
 
 
+def mock_bitquery_api_get_blocks_reward(*args, **kwargs):
+    example_resp_json = {
+        "data": {
+            "ethereum": {
+                "blocks": [
+                    {
+                        "timestamp": {"unixtime": 1708448963},
+                        "reward": 0.024473700594149782,
+                        "address": {
+                            "address": "0x1f9090aae28b8a3dceadf281b0f12828e676c326"
+                        },
+                    },
+                    {
+                        "timestamp": {"unixtime": 1708449035},
+                        "reward": 0.05342909432569912,
+                        "address": {
+                            "address": "0x1f9090aae28b8a3dceadf281b0f12828e676c326"
+                        },
+                    },
+                ]
+            }
+        }
+    }
+
+    blocks = example_resp_json["data"]["ethereum"]["blocks"]
+    return accumulate_blocks_reward(blocks)
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--runapi",
@@ -512,6 +544,14 @@ def patch_etherscan_get_block_api(monkeypatch):
     )
 
 
+@pytest.fixture(scope="function")
+def patch_bitquery_get_blocks_reward(monkeypatch):
+    monkeypatch.setattr(
+        "app.modules.staking.proceeds.service.aggregated.get_blocks_reward",
+        mock_bitquery_api_get_blocks_reward,
+    )
+
+
 @pytest.fixture(scope="function")
 def mock_users_db(app, user_accounts):
     alice = database.user.add_user(user_accounts[0].address)

backend/tests/modules/staking/test_aggegated_staking_proceeds.py

@@ -8,10 +8,12 @@ def before(app):
     pass
 
 
-def test_aggregated_staking_proceeds(patch_etherscan_transactions_api):
+def test_aggregated_staking_proceeds(
+    patch_etherscan_transactions_api, patch_bitquery_get_blocks_reward
+):
     context = get_context(1)
 
     service = AggregatedStakingProceeds()
     result = service.get_staking_proceeds(context)
 
-    assert result == 68311976_811131780
+    assert result == 68311976_889034574