convert eth to wei, exclude last block with reward, general refactor

backend/app/context/epoch_details.py

@@ -3,6 +3,7 @@
 
 from app.context.epoch_state import EpochState
 from app.context.helpers import check_if_future
+from app.exceptions import WrongBlocksRange
 from app.extensions import epochs
 from app.infrastructure import graphql
 from app.infrastructure.external_api.etherscan.blocks import get_block_num_from_ts
@@ -50,6 +51,12 @@ def __init__(
     def duration_range(self) -> tuple[int, int]:
         return self.start_sec, self.end_sec
 
+    @property
+    def no_blocks(self):
+        if not self.end_block or not self.start_block:
+            raise WrongBlocksRange
+        return self.end_block - self.start_block
+
 
 def get_epoch_details(epoch_num: int, epoch_state: EpochState) -> EpochDetails:
     if epoch_state == EpochState.FUTURE:

backend/app/exceptions.py

@@ -238,3 +238,11 @@ class EmptyAllocations(OctantException):
 
     def __init__(self):
         super().__init__(self.description, self.code)
+
+
+class WrongBlocksRange(OctantException):
+    code = 400
+    description = "Attempt to use wrong range of start and end block in epoch"
+
+    def __init__(self):
+        super().__init__(self.description, self.code)

backend/app/infrastructure/external_api/bitquery/blocks_reward.py

@@ -1,30 +1,35 @@
 import requests
 
+
 import app as app_module
 from app.constants import BITQUERY_API
 from app.exceptions import ExternalApiException
-from app.extensions import w3
 from app.infrastructure.external_api.bitquery.req_producer import (
     produce_payload,
     BitQueryActions,
     get_bitquery_header,
 )
 
 
-def accumulate_blocks_reward_wei(blocks: list) -> int:
-    blocks_reward_gwei = 0.0
-    for block in blocks:
-        blocks_reward_gwei += float(block["reward"])
-
-    return int(w3.to_wei(blocks_reward_gwei, "gwei"))
-
+def get_blocks_rewards(
+    address: str, start_time: str, end_time: str, limit: int
+) -> list:
+    """
+    Fetch Ethereum blocks within a specified time range in ascending order by timestamp.
 
-def get_blocks_reward(address: str, start_time: str, end_time: str) -> int:
+    Args:
+    - start_time (str): The start time in ISO 8601 format.
+    - end_time (str): The end time in ISO 8601 format.
+    - address (str): The miner (fee recipient) address.
+    - limit (int): The number of blocks to retrieve starting from start_time.
+                   Useful whilst getting end_blocks exclusively from epochs.
+    """
     payload = produce_payload(
         action_type=BitQueryActions.GET_BLOCK_REWARDS,
         address=address,
         start_time=start_time,
         end_time=end_time,
+        limit=limit,
     )
     headers = get_bitquery_header()
 
@@ -39,5 +44,4 @@ def get_blocks_reward(address: str, start_time: str, end_time: str) -> int:
         raise ExternalApiException(api_url, e, 500)
 
     blocks = json_response.json()["data"]["ethereum"]["blocks"]
-    blocks_reward = accumulate_blocks_reward_wei(blocks)
-    return blocks_reward
+    return blocks

backend/app/infrastructure/external_api/bitquery/req_producer.py

@@ -25,13 +25,13 @@ def produce_payload(action_type: BitQueryActions, **query_values) -> str:
 
 
 def _block_rewards_payload(
-    start_time: str, end_time: str, address: str, **kwargs
+    start_time: str, end_time: str, address: str, limit: int, **kwargs
 ) -> str:
     payload = json.dumps(
         {
             "query": f"""query ($network: EthereumNetwork!, $from: ISO8601DateTime, $till: ISO8601DateTime) {{
           ethereum(network: $network) {{
-            blocks(time: {{since: $from, till: $till}}) {{
+            blocks(time: {{since: $from, till: $till}}, options: {{asc: "timestamp.unixtime", limit: {limit}}}) {{
               timestamp {{
                 unixtime
               }}
@@ -47,6 +47,7 @@ def _block_rewards_payload(
                     "network": "ethereum",
                     "from": start_time,
                     "till": end_time,
+                    "limit": limit,
                     "dateFormat": "%Y-%m-%d",
                 }
             ),

backend/app/legacy/utils/time.py

@@ -19,6 +19,9 @@ def timestamp_s(self) -> float:
     def datetime(self) -> DateTime:
         return DateTime.fromtimestamp(self.timestamp_s())
 
+    def to_isoformat(self):
+        return self.datetime().isoformat()
+
     def __eq__(self, o):
         if isinstance(o, Timestamp):
             return self._timestamp_us == o._timestamp_us
@@ -60,3 +63,8 @@ def sec_to_days(sec: int) -> int:
 
 def days_to_sec(days: int) -> int:
     return int(days * 86400)
+
+
+def timestamp_to_isoformat(timestamp_sec: int) -> str:
+    timestamp = from_timestamp_s(timestamp_sec)
+    return timestamp.to_isoformat()

backend/app/modules/staking/proceeds/core.py

@@ -1,3 +1,5 @@
+from decimal import Decimal
+
 import pandas as pd
 from gmpy2 import mpz
 
@@ -54,8 +56,15 @@ def sum_withdrawals(withdrawals_txs: list[dict]) -> int:
     return w3.to_wei(int(total_gwei), "gwei")
 
 
-def aggregate_proceeds(mev: int, withdrawals: int, blocks_reward: int) -> int:
-    return mev + withdrawals + blocks_reward
+def sum_blocks_rewards(blocks_rewards: list) -> int:
+    df = pd.DataFrame(blocks_rewards)
+    blocks_reward_eth = df["reward"].apply(Decimal).sum()
+
+    return int(w3.to_wei(blocks_reward_eth, "ether"))
+
+
+def aggregate_proceeds(mev: int, withdrawals: int, blocks_rewards: list) -> int:
+    return mev + withdrawals + sum_blocks_rewards(blocks_rewards)
 
 
 def _filter_deposit_withdrawals(amount: mpz) -> mpz:

backend/app/modules/staking/proceeds/service/aggregated.py

@@ -5,8 +5,8 @@
     get_transactions,
     AccountAction,
 )
-from app.infrastructure.external_api.bitquery.blocks_reward import get_blocks_reward
-from app.modules.common.timestamp_converter import timestamp_to_isoformat
+from app.infrastructure.external_api.bitquery.blocks_reward import get_blocks_rewards
+from app.legacy.utils.time import timestamp_to_isoformat
 from app.modules.staking.proceeds.core import (
     sum_mev,
     sum_withdrawals,
@@ -16,23 +16,23 @@
 
 
 class AggregatedStakingProceeds(Model):
-    def _compute_blocks_reward(
-        self, start_sec: int, end_sec: int, withdrawals_target: str
-    ) -> int:
-        blocks_reward = 0
+    def _retrieve_blocks_rewards(
+        self, start_sec: int, end_sec: int, withdrawals_target: str, limit: int
+    ) -> list:
+        blocks_rewards = []
 
         if end_sec is None:
-            return blocks_reward
+            return blocks_rewards
 
         start_datetime, end_datetime = (
             timestamp_to_isoformat(start_sec),
             timestamp_to_isoformat(end_sec),
         )
 
-        blocks_reward = get_blocks_reward(
-            withdrawals_target, start_datetime, end_datetime
+        blocks_rewards = get_blocks_rewards(
+            withdrawals_target, start_datetime, end_datetime, limit=limit
         )
-        return blocks_reward
+        return blocks_rewards
 
     def get_staking_proceeds(self, context: Context) -> int:
         """
@@ -51,28 +51,33 @@ def get_staking_proceeds(self, context: Context) -> int:
             context.epoch_details.end_block,
         )
 
-        if end_block is not None:
-            end_block -= 1
+        start_sec, end_sec = context.epoch_details.duration_range
+        no_blocks_to_get = context.epoch_details.no_blocks
+
+        blocks_rewards = self._retrieve_blocks_rewards(
+            start_sec, end_sec, withdrawals_target, limit=no_blocks_to_get
+        )
 
+        end_block_for_transactions = end_block - 1
         normal = get_transactions(
-            withdrawals_target, start_block, end_block, tx_type=AccountAction.NORMAL
+            withdrawals_target,
+            start_block,
+            end_block_for_transactions,
+            tx_type=AccountAction.NORMAL,
         )
         internal = get_transactions(
-            withdrawals_target, start_block, end_block, tx_type=AccountAction.INTERNAL
+            withdrawals_target,
+            start_block,
+            end_block_for_transactions,
+            tx_type=AccountAction.INTERNAL,
         )
         withdrawals = get_transactions(
             withdrawals_target,
             start_block,
-            end_block,
+            end_block_for_transactions,
             tx_type=AccountAction.BEACON_WITHDRAWAL,
         )
         mev_value = sum_mev(withdrawals_target, normal, internal)
         withdrawals_value = sum_withdrawals(withdrawals)
 
-        start_sec, end_sec = context.epoch_details.duration_range
-
-        blocks_reward = self._compute_blocks_reward(
-            start_sec, end_sec, withdrawals_target
-        )
-
-        return aggregate_proceeds(mev_value, withdrawals_value, blocks_reward)
+        return aggregate_proceeds(mev_value, withdrawals_value, blocks_rewards)

backend/tests/conftest.py

@@ -22,9 +22,6 @@
 from app.infrastructure.contracts.erc20 import ERC20
 from app.infrastructure.contracts.proposals import Proposals
 from app.infrastructure.contracts.vault import Vault
-from app.infrastructure.external_api.bitquery.blocks_reward import (
-    accumulate_blocks_reward_wei,
-)
 from app.legacy.controllers.allocations import allocate, deserialize_payload
 from app.legacy.core.allocations import Allocation, AllocationRequest
 from app.legacy.crypto.account import Account as CryptoAccount
@@ -110,7 +107,7 @@ def mock_etherscan_api_get_block_num_from_ts(*args, **kwargs):
     return int(example_resp_json["result"])
 
 
-def mock_bitquery_api_get_blocks_reward(*args, **kwargs):
+def mock_bitquery_api_get_blocks_rewards(*args, **kwargs):
     example_resp_json = {
         "data": {
             "ethereum": {
@@ -135,7 +132,7 @@ def mock_bitquery_api_get_blocks_reward(*args, **kwargs):
     }
 
     blocks = example_resp_json["data"]["ethereum"]["blocks"]
-    return accumulate_blocks_reward_wei(blocks)
+    return blocks
 
 
 def pytest_addoption(parser):
@@ -545,10 +542,10 @@ def patch_etherscan_get_block_api(monkeypatch):
 
 
 @pytest.fixture(scope="function")
-def patch_bitquery_get_blocks_reward(monkeypatch):
+def patch_bitquery_get_blocks_rewards(monkeypatch):
     monkeypatch.setattr(
-        "app.modules.staking.proceeds.service.aggregated.get_blocks_reward",
-        mock_bitquery_api_get_blocks_reward,
+        "app.modules.staking.proceeds.service.aggregated.get_blocks_rewards",
+        mock_bitquery_api_get_blocks_rewards,
     )
 
 

backend/tests/helpers/context.py

@@ -6,14 +6,22 @@
 
 
 def get_epoch_details(
-    epoch_num: int, start=1000, duration=1000, decision_window=500, remaining_sec=1000
+    epoch_num: int,
+    start=1000,
+    duration=1000,
+    decision_window=500,
+    remaining_sec=1000,
+    start_block: int = 12712551,
+    end_block: int = 12712551,
 ):
     return EpochDetails(
         epoch_num=epoch_num,
         duration=duration,
         start=start,
         decision_window=decision_window,
         remaining_sec=remaining_sec,
+        start_block=start_block,
+        end_block=end_block,
     )
 
 

backend/tests/infrastracture/external_api/bitquery/test_acc_blocks_reward.py

@@ -1,13 +1,18 @@
 import pytest
-from app.infrastructure.external_api.bitquery.blocks_reward import (
-    accumulate_blocks_reward_wei,
+from app.modules.staking.proceeds.core import (
+    sum_blocks_rewards,
 )
 
 
 @pytest.mark.parametrize(
     "blocks, result",
-    [([{"reward": 0.024473700594149782}, {"reward": 0.05342909432569912}], 77902794)],
+    [
+        (
+            [{"reward": 0.024473700594149782}, {"reward": 0.05342909432569912}],
+            77902794919848899,
+        )
+    ],
 )
-def test_accumulate_blocks_reward_wei(blocks, result):
-    actual_result = accumulate_blocks_reward_wei(blocks)
+def test_sum_blocks_rewards(blocks, result):
+    actual_result = sum_blocks_rewards(blocks)
     assert actual_result == result

backend/tests/modules/staking/test_aggegated_staking_proceeds.py

@@ -9,11 +9,17 @@ def before(app):
 
 
 def test_aggregated_staking_proceeds(
-    patch_etherscan_transactions_api, patch_bitquery_get_blocks_reward
+    patch_etherscan_transactions_api, patch_bitquery_get_blocks_rewards
 ):
+    """
+    Expected results for the test:
+    MEV 66813166811131780
+    WITHDRAWALS 1498810000000000
+    BLOCKS REWARD 77902794919848899
+    """
     context = get_context(1)
 
     service = AggregatedStakingProceeds()
     result = service.get_staking_proceeds(context)
 
-    assert result == 68311976_889034574
+    assert result == 146214771_730980679