From e2ea9bffa4c31ba358b4f33099ffb59ef03ee92b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20=C5=BBelazko?= Date: Tue, 5 Sep 2023 13:44:51 +0000 Subject: [PATCH] OCT-579: add-pagination-to-history --- backend/app/controllers/history.py | 77 ++--- backend/app/core/history.py | 68 ++-- backend/app/core/pagination.py | 61 ++++ backend/app/database/allocations.py | 35 +- backend/app/infrastructure/graphql/locks.py | 102 +++--- backend/app/infrastructure/graphql/unlocks.py | 99 +++--- .../app/infrastructure/graphql/withdrawals.py | 66 +++- backend/app/infrastructure/routes/history.py | 27 +- backend/app/utils/__init__.py | 0 backend/app/utils/timestamp.py | 45 +++ backend/tests/conftest.py | 2 +- .../{ => database}/test_allocations_db.py | 0 .../tests/{ => database}/test_claims_db.py | 0 backend/tests/test_history.py | 313 ++++++++++++++++-- flake.nix | 1 + 15 files changed, 695 insertions(+), 201 deletions(-) create mode 100644 backend/app/core/pagination.py create mode 100644 backend/app/utils/__init__.py create mode 100644 backend/app/utils/timestamp.py rename backend/tests/{ => database}/test_allocations_db.py (100%) rename backend/tests/{ => database}/test_claims_db.py (100%) diff --git a/backend/app/controllers/history.py b/backend/app/controllers/history.py index dc720f3a35..3062d686c8 100644 --- a/backend/app/controllers/history.py +++ b/backend/app/controllers/history.py @@ -1,9 +1,11 @@ from dataclasses import dataclass from enum import StrEnum -from typing import List +from typing import List, Tuple, Optional +from operator import attrgetter from dataclass_wizard import JSONWizard from app.core import history +from app.core.pagination import Paginator, Cursor class OpType(StrEnum): @@ -20,43 +22,36 @@ class HistoryEntry(JSONWizard): timestamp: int # Should be in microseconds precision -def user_history(user_address: str) -> List[HistoryEntry]: - allocations = [ - HistoryEntry( - type=OpType.ALLOCATION, - amount=r.amount, - timestamp=r.timestamp, - ) - for r in history.get_allocations(user_address, 0) - ] - - locks = [ - HistoryEntry( - type=OpType.LOCK, - amount=r.amount, - timestamp=r.timestamp, - ) - for r in history.get_locks(user_address, 0) - ] - - unlocks = [ - HistoryEntry( - type=OpType.UNLOCK, - amount=r.amount, - timestamp=r.timestamp, - ) - for r in history.get_unlocks(user_address, 0) - ] - - withdrawals = [ - HistoryEntry( - type=OpType.WITHDRAWAL, - amount=r.amount, - timestamp=r.timestamp, - ) - for r in history.get_withdrawals(user_address, 0) - ] - - combined = allocations + locks + unlocks + withdrawals - - return sorted(combined, key=lambda x: x.timestamp, reverse=True) +def user_history( + user_address: str, cursor: str = None, limit: int = 20 +) -> Tuple[List[HistoryEntry], Optional[str]]: + limit = limit if limit < 100 else 100 + + (from_timestamp, offset_at_timestamp) = Cursor.decode(cursor) + query_limit = Paginator.query_limit(limit, offset_at_timestamp) + + all = _collect_history_records(user_address, from_timestamp, query_limit) + return Paginator.extract_page(all, offset_at_timestamp, limit) + + +def _collect_history_records( + user_address, from_timestamp, query_limit +) -> List[HistoryEntry]: + events = [] + for event_getter, event_type in [ + (history.get_allocations, OpType.ALLOCATION), + (history.get_locks, OpType.LOCK), + (history.get_unlocks, OpType.UNLOCK), + (history.get_withdrawals, OpType.WITHDRAWAL), + ]: + events += [ + HistoryEntry( + type=event_type, + amount=e.amount, + timestamp=e.timestamp.timestamp_us(), + ) + for e in event_getter(user_address, from_timestamp, query_limit) + ] + + sort_keys = attrgetter("timestamp", "type", "amount") + return sorted(events, key=sort_keys, reverse=True) diff --git a/backend/app/core/history.py b/backend/app/core/history.py index 187774385f..673256da27 100644 --- a/backend/app/core/history.py +++ b/backend/app/core/history.py @@ -4,10 +4,15 @@ from typing import List from app.database import allocations +from app.utils.timestamp import ( + Timestamp, + from_datetime, + from_timestamp_s, + from_timestamp_us, + now, +) -from app.infrastructure.graphql.locks import get_locks_by_address -from app.infrastructure.graphql.unlocks import get_unlocks_by_address -from app.infrastructure.graphql.withdrawals import get_withdrawals_by_address_and_ts +from app.infrastructure.graphql import locks, unlocks, withdrawals class OpType(StrEnum): @@ -19,7 +24,7 @@ class OpType(StrEnum): class LockItem: type: OpType amount: int - timestamp: int # Should be in microseconds + timestamp: Timestamp @dataclass(frozen=True) @@ -27,67 +32,72 @@ class AllocationItem: address: str epoch: int amount: int - timestamp: int # Should be in microseconds + timestamp: Timestamp @dataclass(frozen=True) class WithdrawalItem: amount: int address: str - timestamp: int # Should be in microseconds + timestamp: Timestamp -def get_locks(user_address: str, from_timestamp_us: int) -> List[LockItem]: +def get_locks( + user_address: str, from_timestamp: Timestamp, limit: int +) -> List[LockItem]: return [ LockItem( type=OpType.LOCK, amount=int(r["amount"]), - timestamp=_seconds_to_microseconds(r["timestamp"]), + timestamp=from_timestamp_s(r["timestamp"]), + ) + for r in locks.get_user_locks_history( + user_address, int(from_timestamp.timestamp_s()), limit ) - for r in get_locks_by_address(user_address) - if int(r["timestamp"]) >= from_timestamp_us ] -def get_unlocks(user_address: str, from_timestamp_us: int) -> List[LockItem]: +def get_unlocks( + user_address: str, from_timestamp: Timestamp, limit: int +) -> List[LockItem]: return [ LockItem( type=OpType.UNLOCK, amount=int(r["amount"]), - timestamp=_seconds_to_microseconds(r["timestamp"]), + timestamp=from_timestamp_s(r["timestamp"]), + ) + for r in unlocks.get_user_unlocks_history( + user_address, int(from_timestamp.timestamp_s()), limit ) - for r in get_unlocks_by_address(user_address) - if int(r["timestamp"]) >= from_timestamp_us ] -def get_allocations(user_address: str, from_timestamp_us: int) -> List[AllocationItem]: +def get_allocations( + user_address: str, from_timestamp: Timestamp, limit: int +) -> List[AllocationItem]: return [ AllocationItem( address=r.proposal_address, epoch=r.epoch, amount=int(r.amount), - timestamp=_datetime_to_microseconds(r.created_at), + timestamp=from_datetime(r.created_at), + ) + for r in allocations.get_user_allocations_history( + user_address, from_timestamp.datetime(), limit ) - for r in allocations.get_all_by_user(user_address, with_deleted=True) - if _datetime_to_microseconds(r.created_at) >= from_timestamp_us ] -def get_withdrawals(user_address: str, from_timestamp_s: int) -> List[WithdrawalItem]: +def get_withdrawals( + user_address: str, from_timestamp: Timestamp, limit: int +) -> List[WithdrawalItem]: return [ WithdrawalItem( address=r["user"], amount=int(r["amount"]), - timestamp=_seconds_to_microseconds(r["timestamp"]), + timestamp=from_timestamp_s(r["timestamp"]), + ) + for r in withdrawals.get_user_withdrawals_history( + user_address, int(from_timestamp.timestamp_s()), limit ) - for r in get_withdrawals_by_address_and_ts(user_address, from_timestamp_s) ] - - -def _datetime_to_microseconds(date: datetime.datetime) -> int: - return int(date.timestamp() * 10**6) - - -def _seconds_to_microseconds(timestamp: int) -> int: - return timestamp * 10**6 diff --git a/backend/app/core/pagination.py b/backend/app/core/pagination.py new file mode 100644 index 0000000000..9c54b078ef --- /dev/null +++ b/backend/app/core/pagination.py @@ -0,0 +1,61 @@ +import base64 +from typing import List, Tuple, Optional + +from app.utils.timestamp import now, from_timestamp_us, Timestamp + + +class Cursor: + @staticmethod + def encode(timestamp: int, offset: int) -> str: + return base64.urlsafe_b64encode(bytes(f"{timestamp}.{offset}", "ascii")).decode( + "ascii" + ) + + @staticmethod + def decode(cursor: Optional[str]) -> Tuple[Timestamp, int]: + if cursor is None: + return now(), 0 + timestamp, offset = base64.urlsafe_b64decode(cursor).decode("ascii").split(".") + return from_timestamp_us(int(timestamp)), int(offset) + + +class Paginator: + @staticmethod + def query_limit(limit, offset): + # we must query for all elems at the current timestamp (also events from the prev page, thus we add offset) + # and one elem more to get calculate next page cursor + return limit + offset + 1 + + @staticmethod + def extract_page( + all: List, offset_at_timestamp: int, limit: int + ) -> Tuple[List, Optional[str]]: + next_page_start_index = offset_at_timestamp + limit + current_page = all[offset_at_timestamp:next_page_start_index] + + next_page_start_elem = ( + all[next_page_start_index] if next_page_start_index < len(all) else None + ) + next_page_cursor = None + if next_page_start_elem is not None: + next_offset = Paginator._get_offset( + current_page, next_page_start_elem.timestamp, offset_at_timestamp + ) + next_page_cursor = Cursor.encode( + next_page_start_elem.timestamp, next_offset + ) + + return current_page, next_page_cursor + + @staticmethod + def _get_offset(current_page_elems, next_elem_timestamp, prev_offset): + offset = 0 + for elem in current_page_elems[::-1]: + if elem.timestamp == next_elem_timestamp: + offset += 1 + else: + return offset + + # If we reach this line, + # then all elems in the next page have same timestamp, so we have to add offset to the prev_offset + return offset + prev_offset diff --git a/backend/app/database/allocations.py b/backend/app/database/allocations.py index bbfdd862cc..c3e171a5c1 100644 --- a/backend/app/database/allocations.py +++ b/backend/app/database/allocations.py @@ -4,6 +4,7 @@ from eth_utils import to_checksum_address from sqlalchemy.orm import Query +from sqlalchemy import func from app.core.common import AccountFunds from app.database.models import Allocation, User @@ -20,18 +21,38 @@ def get_all_by_epoch(epoch: int, with_deleted=False) -> List[Allocation]: return query.all() -def get_all_by_user(user_address: str, with_deleted=False) -> List[Allocation]: +def get_user_allocations_history( + user_address: str, from_datetime: datetime, limit: int +) -> List[Allocation]: user: User = get_by_address(user_address) if user is None: return [] - query: Query = Allocation.query.filter_by(user_id=user.id) + user_allocations: Query = ( + Allocation.query.filter( + Allocation.user_id == user.id, Allocation.created_at <= from_datetime + ) + .order_by(Allocation.created_at.desc()) + .limit(limit) + .subquery() + ) - if not with_deleted: - query = query.filter(Allocation.deleted_at.is_(None)) + timestamp_at_limit_query = ( + db.session.query( + func.min(user_allocations.c.created_at).label("limit_timestamp") + ) + .group_by(user_allocations.c.user_id) + .subquery() + ) - return query.all() + allocations = Allocation.query.filter( + Allocation.user_id == user.id, + Allocation.created_at <= from_datetime, + Allocation.created_at >= timestamp_at_limit_query.c.limit_timestamp, + ).order_by(Allocation.created_at.desc()) + + return allocations.all() def get_all_by_user_addr_and_epoch( @@ -106,7 +127,7 @@ def get_alloc_sum_by_epoch(epoch: int) -> int: def add_all(epoch: int, user_id: int, allocations): - now = datetime.now() + now = datetime.utcnow() new_allocations = [ Allocation( @@ -126,7 +147,7 @@ def soft_delete_all_by_epoch_and_user_id(epoch: int, user_id: int): epoch=epoch, user_id=user_id, deleted_at=None ).all() - now = datetime.now() + now = datetime.utcnow() for allocation in existing_allocations: allocation.deleted_at = now diff --git a/backend/app/infrastructure/graphql/locks.py b/backend/app/infrastructure/graphql/locks.py index e2c44ae39a..0e488a9226 100644 --- a/backend/app/infrastructure/graphql/locks.py +++ b/backend/app/infrastructure/graphql/locks.py @@ -2,23 +2,46 @@ from gql import gql -def get_locks_by_address(user_address: str): +def get_user_locks_history(user_address: str, from_timestamp: int, limit: int): query = gql( """ - query GetLocks($userAddress: Bytes!) { - lockeds(orderBy: timestamp, where: { user: $userAddress }) { - __typename - amount - timestamp - } + query GetLocks($userAddress: Bytes!, $fromTimestamp: Int!, $limit: Int!) { + lockeds( + orderBy: timestamp + orderDirection: desc + where: {user: $userAddress, timestamp_lte: $fromTimestamp} + first: $limit + ) { + __typename + amount + timestamp + } } - """ + """ ) - variables = {"userAddress": user_address} + variables = { + "userAddress": user_address, + "fromTimestamp": from_timestamp, + "limit": limit, + } app.logger.debug(f"[Subgraph] Getting user {user_address} locks") - result = request_context.graphql_client.execute(query, variable_values=variables)[ - "lockeds" - ] + + partial_result = request_context.graphql_client.execute( + query, variable_values=variables + )["lockeds"] + + result = [] + + if len(partial_result) > 0: + limit_timestamp = partial_result[-1]["timestamp"] + events_at_timestamp_limit = get_locks_by_address_and_timestamp_range( + user_address, limit_timestamp, limit_timestamp + 1 + ) + result_without_events_at_timestamp_limit = list( + filter(lambda x: x["timestamp"] != limit_timestamp, partial_result) + ) + result = result_without_events_at_timestamp_limit + events_at_timestamp_limit + app.logger.debug(f"[Subgraph] Received locks: {result}") return result @@ -28,21 +51,18 @@ def get_locks_by_timestamp_range(from_ts: int, to_ts: int): query = gql( """ query GetLocks($fromTimestamp: Int!, $toTimestamp: Int!) { - lockeds( - orderBy: timestamp, - where: { - timestamp_gte: $fromTimestamp, - timestamp_lt: $toTimestamp - } - ) { - __typename - depositBefore - amount - timestamp - user - } -} - """ + lockeds( + orderBy: timestamp + where: {timestamp_gte: $fromTimestamp, timestamp_lt: $toTimestamp} + ) { + __typename + depositBefore + amount + timestamp + user + } + } + """ ) variables = { @@ -64,22 +84,18 @@ def get_locks_by_address_and_timestamp_range( query = gql( """ query GetLocks($userAddress: Bytes!, $fromTimestamp: Int!, $toTimestamp: Int!) { - lockeds( - orderBy: timestamp, - where: { - timestamp_gte: $fromTimestamp, - timestamp_lt: $toTimestamp, - user: $userAddress - } - ) { - __typename - depositBefore - amount - timestamp - user - } -} - """ + lockeds( + orderBy: timestamp + where: {timestamp_gte: $fromTimestamp, timestamp_lt: $toTimestamp, user: $userAddress} + ) { + __typename + depositBefore + amount + timestamp + user + } + } + """ ) variables = { diff --git a/backend/app/infrastructure/graphql/unlocks.py b/backend/app/infrastructure/graphql/unlocks.py index 2bfc2dcc63..f5377cbbf6 100644 --- a/backend/app/infrastructure/graphql/unlocks.py +++ b/backend/app/infrastructure/graphql/unlocks.py @@ -2,25 +2,47 @@ from gql import gql -def get_unlocks_by_address(user_address): +def get_user_unlocks_history(user_address: str, from_timestamp: int, limit: int): query = gql( """ - query GetLocks($userAddress: Bytes!) { - unlockeds(orderBy: timestamp, where: { user: $userAddress }) { - __typename - amount - timestamp - } + query GetUnlocks($userAddress: Bytes!, $fromTimestamp: Int!, $limit: Int!) { + unlockeds( + orderBy: timestamp + orderDirection: desc + where: {user: $userAddress, timestamp_lte: $fromTimestamp} + first: $limit + ) { + __typename + amount + timestamp + } } """ ) - variables = {"userAddress": user_address} + variables = { + "userAddress": user_address, + "fromTimestamp": from_timestamp, + "limit": limit, + } app.logger.debug(f"[Subgraph] Getting user {user_address} unlocks") - result = request_context.graphql_client.execute(query, variable_values=variables)[ - "unlockeds" - ] + partial_result = request_context.graphql_client.execute( + query, variable_values=variables + )["unlockeds"] + + result = [] + + if len(partial_result) > 0: + limit_timestamp = partial_result[-1]["timestamp"] + events_at_timestamp_limit = get_unlocks_by_address_and_timestamp_range( + user_address, limit_timestamp, limit_timestamp + 1 + ) + result_without_events_at_timestamp_limit = list( + filter(lambda x: x["timestamp"] != limit_timestamp, partial_result) + ) + result = result_without_events_at_timestamp_limit + events_at_timestamp_limit + app.logger.debug(f"[Subgraph] Received unlocks: {result}") return result @@ -30,21 +52,18 @@ def get_unlocks_by_timestamp_range(from_ts, to_ts): query = gql( """ query GetUnlocks($fromTimestamp: Int!, $toTimestamp: Int!) { - unlockeds( - orderBy: timestamp, - where: { - timestamp_gte: $fromTimestamp, - timestamp_lt: $toTimestamp - } - ) { - __typename - depositBefore - amount - timestamp - user - } -} - """ + unlockeds( + orderBy: timestamp + where: {timestamp_gte: $fromTimestamp, timestamp_lt: $toTimestamp} + ) { + __typename + depositBefore + amount + timestamp + user + } + } + """ ) variables = { @@ -69,22 +88,18 @@ def get_unlocks_by_address_and_timestamp_range( query = gql( """ query GetUnlocks($userAddress: Bytes!, $fromTimestamp: Int!, $toTimestamp: Int!) { - unlockeds( - orderBy: timestamp, - where: { - timestamp_gte: $fromTimestamp, - timestamp_lt: $toTimestamp, - user: $userAddress - } - ) { - __typename - depositBefore - amount - timestamp - user - } -} - """ + unlockeds( + orderBy: timestamp + where: {timestamp_gte: $fromTimestamp, timestamp_lt: $toTimestamp, user: $userAddress} + ) { + __typename + depositBefore + amount + timestamp + user + } + } + """ ) variables = { diff --git a/backend/app/infrastructure/graphql/withdrawals.py b/backend/app/infrastructure/graphql/withdrawals.py index df956a2372..b067303822 100644 --- a/backend/app/infrastructure/graphql/withdrawals.py +++ b/backend/app/infrastructure/graphql/withdrawals.py @@ -2,31 +2,85 @@ from gql import gql -def get_withdrawals_by_address_and_ts(user_address: str, gt_timestamp: int): +def get_user_withdrawals_history(user_address: str, from_timestamp: int, limit: int): query = gql( """ - query GetWithdrawals($user_address: Bytes!, $timestamp_gt: Int!) { + query GetWithdrawals($userAddress: Bytes!, $fromTimestamp: Int!, $limit: Int!) { withdrawals( orderBy: timestamp orderDirection: desc - where: {user: $user_address, timestamp_gt: $timestamp_gt} + where: {user: $userAddress, timestamp_lte: $fromTimestamp} + first: $limit ) { amount timestamp user } } - """ + """ + ) + + variables = { + "userAddress": user_address, + "fromTimestamp": from_timestamp, + "limit": limit, + } + + app.logger.info( + f"[Subgraph] Getting user {user_address} withdrawals before ts {from_timestamp}" + ) + partial_result = request_context.graphql_client.execute( + query, variable_values=variables + )["withdrawals"] + + result = [] + + if len(partial_result) > 0: + limit_timestamp = partial_result[-1]["timestamp"] + events_at_timestamp_limit = get_withdrawals_by_address_and_timestamp_range( + user_address, limit_timestamp, limit_timestamp + 1 + ) + result_without_events_at_timestamp_limit = list( + filter(lambda x: x["timestamp"] != limit_timestamp, partial_result) + ) + result = result_without_events_at_timestamp_limit + events_at_timestamp_limit + app.logger.info(f"[Subgraph] Received withdrawals: {result}") + + return result + + +def get_withdrawals_by_address_and_timestamp_range( + user_address: str, from_timestamp: int, to_timestamp: int +): + query = gql( + """ + query GetWithdrawals($userAddress: Bytes!, $fromTimestamp: Int!, $toTimestamp: Int!) { + withdrawals( + orderBy: timestamp + where: {user: $userAddress, timestamp_gte: $fromTimestamp, timestamp_lt: $toTimestamp} + ) { + amount + timestamp + user + } + } + """ ) - variables = {"user_address": user_address, "timestamp_gt": gt_timestamp} + variables = { + "userAddress": user_address, + "fromTimestamp": from_timestamp, + "toTimestamp": to_timestamp, + } app.logger.info( - f"[Subgraph] Getting user {user_address} withdrawals after ts {gt_timestamp}" + f"[Subgraph] Getting user {user_address} withdrawals in timestamp range {from_timestamp} - {to_timestamp}" ) + result = request_context.graphql_client.execute(query, variable_values=variables)[ "withdrawals" ] + app.logger.info(f"[Subgraph] Received withdrawals: {result}") return result diff --git a/backend/app/infrastructure/routes/history.py b/backend/app/infrastructure/routes/history.py index 243c8aa252..57d6fc94ba 100644 --- a/backend/app/infrastructure/routes/history.py +++ b/backend/app/infrastructure/routes/history.py @@ -1,5 +1,6 @@ from flask import current_app as app from flask_restx import Namespace, fields +from flask import request from app.controllers import history from app.extensions import api @@ -30,7 +31,8 @@ { "history": fields.List( fields.Nested(history_item), description="History of user actions" - ) + ), + "next_cursor": fields.String(required=False, description="Next page cursor"), }, ) @@ -38,15 +40,30 @@ @ns.route("/") @ns.doc( params={ - "user_address": "User ethereum address in hexadecimal format (case-insensitive, prefixed with 0x)" + "user_address": "User ethereum address in hexadecimal format (case-insensitive, prefixed with 0x)", } ) class History(OctantResource): + @ns.param("cursor", description="History page cursor", _in="query") + @ns.param("limit", description="History page size", _in="query") @ns.marshal_with(user_history_model) @ns.response(200, "User history successfully retrieved") def get(self, user_address): - app.logger.debug(f"Getting user: {user_address} history") - user_history = history.user_history(user_address) + page_cursor = request.args.get("cursor", type=str) + page_limit = request.args.get("limit", type=int) + + app.logger.debug( + f"Getting history for user: {user_address}. Page details:{(page_cursor, page_limit)} " + ) + user_history, next_cursor = history.user_history( + user_address, page_cursor, page_limit + ) app.logger.debug(f"User: {user_address} history: {user_history}") - return {"history": user_history} + response = ( + {"history": user_history, "next_cursor": next_cursor} + if next_cursor is not None + else {"history": user_history} + ) + + return response diff --git a/backend/app/utils/__init__.py b/backend/app/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/app/utils/timestamp.py b/backend/app/utils/timestamp.py new file mode 100644 index 0000000000..4118a30d6b --- /dev/null +++ b/backend/app/utils/timestamp.py @@ -0,0 +1,45 @@ +from datetime import datetime as DateTime + + +class Timestamp: + def __init__(self, timestamp_us): + if timestamp_us is None: + raise ValueError + self._timestamp_us = timestamp_us + + def get(self) -> int: + return self.timestamp_us() + + def timestamp_us(self) -> int: + return self._timestamp_us + + def timestamp_s(self) -> float: + return self.timestamp_us() / 10**6 + + def datetime(self) -> DateTime: + return DateTime.fromtimestamp(self.timestamp_s()) + + def __eq__(self, o): + if isinstance(o, Timestamp): + return self._timestamp_us == o._timestamp_us + elif isinstance(o, int): + return self._timestamp_us == o + else: + return False + + +def from_timestamp_s(timestamp_s: float) -> Timestamp: + return Timestamp(int(timestamp_s * 10**6)) + + +def from_timestamp_us(timestamp_us) -> Timestamp: + return Timestamp(timestamp_us) + + +def from_datetime(dt: DateTime) -> Timestamp: + return from_timestamp_s(dt.timestamp()) + + +def now() -> Timestamp: + now = DateTime.utcnow().timestamp() + return from_timestamp_s(now) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index e179f5353e..b789364a00 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -36,7 +36,7 @@ LOCKED_RATIO = Decimal("0.000022700000000000099999994") TOTAL_REWARDS = 1_917267577_180363384 ALL_INDIVIDUAL_REWARDS = 9134728_767123337 -USER1_ADDRESS = "0xabcdef7890123456789012345678901234567893" +USER1_ADDRESS = "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" USER2_ADDRESS = "0x2345678901234567890123456789012345678904" # Contracts mocks diff --git a/backend/tests/test_allocations_db.py b/backend/tests/database/test_allocations_db.py similarity index 100% rename from backend/tests/test_allocations_db.py rename to backend/tests/database/test_allocations_db.py diff --git a/backend/tests/test_claims_db.py b/backend/tests/database/test_claims_db.py similarity index 100% rename from backend/tests/test_claims_db.py rename to backend/tests/database/test_claims_db.py diff --git a/backend/tests/test_history.py b/backend/tests/test_history.py index 34f8a13133..4d12e87b54 100644 --- a/backend/tests/test_history.py +++ b/backend/tests/test_history.py @@ -1,10 +1,12 @@ import dataclasses +import math from typing import List import pytest from eth_account.signers.local import LocalAccount from app.controllers.allocations import allocate +from app.controllers.history import user_history from app.core.allocations import AllocationRequest from app.core.history import ( get_locks, @@ -14,6 +16,8 @@ get_withdrawals, ) from app.crypto.eip712 import sign, build_allocations_eip712_data +from app.utils.timestamp import from_timestamp_s, now + from tests.conftest import ( create_payload, MOCK_PROPOSALS, @@ -65,17 +69,17 @@ def before( { "type": "lock", "amount": 300000000000000000000, - "timestamp": 1679645700 * 10**6, + "timestamp": from_timestamp_s(1679645700), }, { "type": "unlock", "amount": 400000000000000000000, - "timestamp": 1679645800 * 10**6, + "timestamp": from_timestamp_s(1679645800), }, { "type": "lock", "amount": 500000000000000000000, - "timestamp": 1679645896 * 10**6, + "timestamp": from_timestamp_s(1679645896), }, ], ), @@ -103,17 +107,17 @@ def before( { "type": "lock", "amount": 500000000000000000000, - "timestamp": 1679645900 * 10**6, + "timestamp": from_timestamp_s(1679645900), }, { "type": "lock", "amount": 300000000000000000000, - "timestamp": 1679645910 * 10**6, + "timestamp": from_timestamp_s(1679645910), }, { "type": "unlock", "amount": 400000000000000000000, - "timestamp": 1679645950 * 10**6, + "timestamp": from_timestamp_s(1679645950), }, ], ), @@ -141,17 +145,17 @@ def before( { "type": "unlock", "amount": 400000000000000000000, - "timestamp": 1679645800 * 10**6, + "timestamp": from_timestamp_s(1679645800), }, { "type": "lock", "amount": 500000000000000000000, - "timestamp": 1679645900 * 10**6, + "timestamp": from_timestamp_s(1679645900), }, { "type": "unlock", "amount": 600000000000000000000, - "timestamp": 1679646000 * 10**6, + "timestamp": from_timestamp_s(1679646000), }, ], ), @@ -173,9 +177,12 @@ def test_history_locks(mocker, locks, unlocks, expected_history): user_address = USER1_ADDRESS # Test various functions from core/history - history = get_locks(user_address, 0) + get_unlocks(user_address, 0) + history = get_locks(user_address, from_timestamp_s(1679647000), 100) + get_unlocks( + user_address, from_timestamp_s(1679647000), 100 + ) history = [ - dataclasses.asdict(item) for item in sorted(history, key=lambda x: x.timestamp) + dataclasses.asdict(item) + for item in sorted(history, key=lambda x: x.timestamp.timestamp_us()) ] assert history == expected_history @@ -188,25 +195,25 @@ def test_history_locks(mocker, locks, unlocks, expected_history): [ { "user": "0x1000000000000000000000000000000000000000", - "amount": 100000000000000000000, - "timestamp": 200, + "amount": 200000000000000000000, + "timestamp": 100, }, { "user": "0x1000000000000000000000000000000000000000", - "amount": 200000000000000000000, - "timestamp": 100, + "amount": 100000000000000000000, + "timestamp": 200, }, ], [ { "address": "0x1000000000000000000000000000000000000000", - "amount": 200000000000000000000, - "timestamp": 100 * 10**6, + "amount": 100000000000000000000, + "timestamp": from_timestamp_s(200), }, { "address": "0x1000000000000000000000000000000000000000", - "amount": 100000000000000000000, - "timestamp": 200 * 10**6, + "amount": 200000000000000000000, + "timestamp": from_timestamp_s(100), }, ], ), @@ -222,7 +229,7 @@ def test_history_locks(mocker, locks, unlocks, expected_history): { "address": "0x1000000000000000000000000000000000000000", "amount": 300000000000000000000, - "timestamp": 100 * 10**6, + "timestamp": from_timestamp_s(100), }, ], ), @@ -235,10 +242,11 @@ def test_history_locks(mocker, locks, unlocks, expected_history): def test_history_withdrawals(mocker, withdrawals, expected_history_sorted_by_ts): mock_graphql(mocker, withdrawals_events=withdrawals) - history = get_withdrawals("0x1000000000000000000000000000000000000000", 0) - history = [ - dataclasses.asdict(item) for item in sorted(history, key=lambda x: x.timestamp) - ] + history = get_withdrawals( + "0x1000000000000000000000000000000000000000", from_timestamp_s(300), 100 + ) + + history = [dataclasses.asdict(i) for i in history] assert history == expected_history_sorted_by_ts @@ -261,7 +269,8 @@ def test_history_allocations(proposal_accounts, user_accounts): ) ) - allocs = _allocation_items_to_tuples(get_allocations(user1.address, 0)) + # query timestamp is set to now since allocation flow creates allocations at current timestamp + allocs = _allocation_items_to_tuples(get_allocations(user1.address, now(), 100)) assert _compare_two_unordered_lists( allocs, @@ -284,7 +293,8 @@ def test_history_allocations(proposal_accounts, user_accounts): ) ) - allocs = _allocation_items_to_tuples(get_allocations(user1.address, 0)) + # query timestamp is set to now since allocation flow creates allocations at current timestamp + allocs = _allocation_items_to_tuples(get_allocations(user1.address, now(), 100)) assert _compare_two_unordered_lists( allocs, @@ -311,7 +321,7 @@ def test_history_allocations(proposal_accounts, user_accounts): ) ) - allocs = _allocation_items_to_tuples(get_allocations(user1.address, 0)) + allocs = _allocation_items_to_tuples(get_allocations(user1.address, now(), 100)) assert _compare_two_unordered_lists( allocs, @@ -329,6 +339,255 @@ def test_history_allocations(proposal_accounts, user_accounts): ) +@pytest.mark.parametrize("page_limit", [1, 2, 3, 5, 8, 13]) +@pytest.mark.parametrize( + "deposits, withdrawals, expected_history", + [ + ( # Case 1: long, linear history + [ + { + "__typename": "Locked", + "amount": "500000000000000000000", + "timestamp": 1000000001, + }, + { + "__typename": "Locked", + "amount": "300000000000000000000", + "timestamp": 1000000002, + }, + { + "__typename": "Unlocked", + "amount": "400000000000000000000", + "timestamp": 1000000003, + }, + { + "__typename": "Locked", + "amount": "300000000000000000000", + "timestamp": 1000000004, + }, + { + "__typename": "Unlocked", + "amount": "100000000000000000000", + "timestamp": 1000000006, + }, + { + "__typename": "Unlocked", + "amount": "100000000000000000000", + "timestamp": 1000000007, + }, + ], + [ + { + "user": USER1_ADDRESS, + "amount": 100000000000000000000, + "timestamp": 1000000005, + }, + { + "user": USER1_ADDRESS, + "amount": 100000000000000000000, + "timestamp": 1000000008, + }, + ], + [ + { + "type": "withdrawal", + "amount": 100000000000000000000, + "timestamp": from_timestamp_s(1000000008).timestamp_us(), + }, + { + "type": "unlock", + "amount": 100000000000000000000, + "timestamp": from_timestamp_s(1000000007).timestamp_us(), + }, + { + "type": "unlock", + "amount": 100000000000000000000, + "timestamp": from_timestamp_s(1000000006).timestamp_us(), + }, + { + "type": "withdrawal", + "amount": 100000000000000000000, + "timestamp": from_timestamp_s(1000000005).timestamp_us(), + }, + { + "type": "lock", + "amount": 300000000000000000000, + "timestamp": from_timestamp_s(1000000004).timestamp_us(), + }, + { + "type": "unlock", + "amount": 400000000000000000000, + "timestamp": from_timestamp_s(1000000003).timestamp_us(), + }, + { + "type": "lock", + "amount": 300000000000000000000, + "timestamp": from_timestamp_s(1000000002).timestamp_us(), + }, + { + "type": "lock", + "amount": 500000000000000000000, + "timestamp": from_timestamp_s(1000000001).timestamp_us(), + }, + ], + ), + ( # Case 2: events happening at the same timestamp + [ + { + "__typename": "Locked", + "amount": "500000000000000000000", + "timestamp": 1000000001, + }, + { + "__typename": "Locked", + "amount": "300000000000000000000", + "timestamp": 1000000001, + }, + { + "__typename": "Unlocked", + "amount": "400000000000000000000", + "timestamp": 1000000001, + }, + { + "__typename": "Locked", + "amount": "300000000000000000000", + "timestamp": 1000000001, + }, + { + "__typename": "Unlocked", + "amount": "100000000000000000000", + "timestamp": 1000000001, + }, + { + "__typename": "Unlocked", + "amount": "100000000000000000000", + "timestamp": 1000000001, + }, + ], + [ + { + "user": USER1_ADDRESS, + "amount": 100000000000000000000, + "timestamp": 1000000001, + }, + { + "user": USER1_ADDRESS, + "amount": 100000000000000000000, + "timestamp": 1000000001, + }, + ], + [ + { + "type": "withdrawal", + "amount": 100000000000000000000, + "timestamp": from_timestamp_s(1000000001).timestamp_us(), + }, + { + "type": "withdrawal", + "amount": 100000000000000000000, + "timestamp": from_timestamp_s(1000000001).timestamp_us(), + }, + { + "type": "unlock", + "amount": 400000000000000000000, + "timestamp": from_timestamp_s(1000000001).timestamp_us(), + }, + { + "type": "unlock", + "amount": 100000000000000000000, + "timestamp": from_timestamp_s(1000000001).timestamp_us(), + }, + { + "type": "unlock", + "amount": 100000000000000000000, + "timestamp": from_timestamp_s(1000000001).timestamp_us(), + }, + { + "type": "lock", + "amount": 500000000000000000000, + "timestamp": from_timestamp_s(1000000001).timestamp_us(), + }, + { + "type": "lock", + "amount": 300000000000000000000, + "timestamp": from_timestamp_s(1000000001).timestamp_us(), + }, + { + "type": "lock", + "amount": 300000000000000000000, + "timestamp": from_timestamp_s(1000000001).timestamp_us(), + }, + ], + ), + ], +) +def test_complete_user_history( + mocker, + deposits, + withdrawals, + expected_history, + page_limit, + proposal_accounts, + user_accounts, +): + # given + + mock_graphql(mocker, deposit_events=deposits, withdrawals_events=withdrawals) + + user1 = user_accounts[0] + proposals = proposal_accounts[0:3] + payload = create_payload(proposals, [100, 200, 300]) + signature = sign(user1, build_allocations_eip712_data(payload)) + MOCK_EPOCHS.get_pending_epoch.return_value = 3 + + allocate( + AllocationRequest( + payload=payload, + signature=signature, + override_existing_allocations=True, + ) + ) + + allocations = get_allocations(user1.address, now(), 10) + allocations = sorted( + [ + { + "type": "allocation", + "amount": a.amount, + "timestamp": a.timestamp.timestamp_us(), + } + for a in allocations + ], + key=lambda x: x["amount"], + reverse=True, + ) + + assert len(allocations) == 3 + + expected_history = allocations + expected_history + + # when + + expected_pages_no = math.ceil(len(expected_history) / page_limit) + + history = [] + cursor = None + for i in range(0, expected_pages_no): + curr_page, cursor = user_history(USER1_ADDRESS, cursor, limit=page_limit) + + history += curr_page + + if i + 1 != expected_pages_no: # if not last page + assert len(curr_page) == page_limit + assert cursor is not None + + # then + assert cursor is None + + assert len(history) == len(expected_history) + assert [dataclasses.asdict(r) for r in history] == expected_history + + def _allocation_items_to_tuples(allocation_items: List[AllocationItem]) -> List[tuple]: return [(a.address, a.epoch, a.amount) for a in allocation_items] diff --git a/flake.nix b/flake.nix index f56b693c32..51ca6271b9 100644 --- a/flake.nix +++ b/flake.nix @@ -32,6 +32,7 @@ python yarn16 pkgs.poetry + pkgs.envsubst # (pkgs.poetry.override { python3 = pkgs.python311; }) ] ++ darwinInputs; };