From 60cd7e216dbf2cf9f246fccb1df08b76770069ab Mon Sep 17 00:00:00 2001 From: Pawel Peregud Date: Mon, 23 Dec 2024 09:35:24 +0100 Subject: [PATCH 1/6] apitest: potential fix for nonce-too-low problem Fixes OCT-1822. Potentially helps with apitest flakiness. --- backend/app/infrastructure/contracts/erc20.py | 4 ++-- backend/v2/glms/contracts.py | 14 -------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/backend/app/infrastructure/contracts/erc20.py b/backend/app/infrastructure/contracts/erc20.py index 350f3ccd6c..00ceae1d46 100644 --- a/backend/app/infrastructure/contracts/erc20.py +++ b/backend/app/infrastructure/contracts/erc20.py @@ -14,7 +14,7 @@ def glm_fund(self, to_address, nonce): return self.w3.eth.send_raw_transaction(signed_tx.rawTransaction) def transfer(self, sender, receiver: str, amount: int): - nonce = self.w3.eth.get_transaction_count(sender.address) + nonce = self.w3.eth.get_transaction_count(sender.address, "pending") transaction = self.contract.functions.transfer( receiver, amount ).build_transaction({"from": sender.address, "nonce": nonce}) @@ -26,7 +26,7 @@ def approve(self, owner, benefactor, wad: int): print("owner address: ", owner.address) print("owner key: ", owner.key) print("benefactor of lock: ", benefactor) - nonce = self.w3.eth.get_transaction_count(owner.address) + nonce = self.w3.eth.get_transaction_count(owner.address, "pending") transaction = self.contract.functions.approve( benefactor, wad ).build_transaction({"from": owner.address, "nonce": nonce}) diff --git a/backend/v2/glms/contracts.py b/backend/v2/glms/contracts.py index f5e9b26273..4a167009be 100644 --- a/backend/v2/glms/contracts.py +++ b/backend/v2/glms/contracts.py @@ -9,17 +9,6 @@ class AddressKey(Protocol): class GLMContracts(SmartContract): - # def glm_fund(self, to_address, nonce): - # transaction = self.contract.functions.transfer( - # to_address, app.config["GLM_WITHDRAWAL_AMOUNT"] - # ).build_transaction({"from": app.config["GLM_SENDER_ADDRESS"], "nonce": nonce}) - # signed_tx = self.w3.eth.account.sign_transaction( - # transaction, app.config["GLM_SENDER_PRIVATE_KEY"] - # ) - # return self.w3.eth.send_raw_transaction(signed_tx.rawTransaction) - - # def transfer(self, sender, receiver: str, amount: int): - # async def transfer(self, sender_address: str, receiver: str, amount: int): async def transfer( self, sender: AddressKey, receiver_address: str, amount: int ) -> None: @@ -42,9 +31,6 @@ async def approve(self, owner: AddressKey, benefactor_address, wad: int): signed_tx = self.w3.eth.account.sign_transaction(transaction, owner.key) return self.w3.eth.send_raw_transaction(signed_tx.rawTransaction) - # def balance_of(self, owner: str) -> int: - # return self.contract.functions.balanceOf(owner).call() - ERC20_ABI = [ { From a394f7a012b568daf5f749fbca2bd3156f20b612 Mon Sep 17 00:00:00 2001 From: Pawel Peregud Date: Mon, 23 Dec 2024 10:34:09 +0100 Subject: [PATCH 2/6] apitest: collect more info on /snapshots/pending responces --- backend/tests/conftest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index b39c1cb11e..97bac59064 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -614,8 +614,11 @@ def snapshot_status(self, epoch): return json.loads(rv.text), rv.status_code def pending_snapshot(self): - rv = self._flask_client.post("/snapshots/pending").text - return json.loads(rv) + rv = self._flask_client.post("/snapshots/pending") + current_app.logger.debug( + f"Request to /snapshots/pending [{rv.status_code}] returned text {rv.text}" + ) + return json.loads(rv.text) def pending_snapshot_simulate(self): rv = self._flask_client.get("/snapshots/pending/simulate") From d96d97ab0bd0768b47f512bb6903ab5b0640ee09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20S=C5=82omnicki?= Date: Wed, 8 Jan 2025 15:57:54 +0100 Subject: [PATCH 3/6] caqd-369: update backend postgresql max_connections --- ci/argocd/templates/octant-application.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/argocd/templates/octant-application.yaml b/ci/argocd/templates/octant-application.yaml index f26c182d04..4fea8b0f97 100644 --- a/ci/argocd/templates/octant-application.yaml +++ b/ci/argocd/templates/octant-application.yaml @@ -58,6 +58,10 @@ spec: value: '${BACKEND_SERVER_REPLICAS:-3}' - name: backendPgsql.deploy value: '${BACKEND_SERVER_PGSQL:-true}' + - name: backendPgsql.primary.extraEnvVars[0].name + value: POSTGRESQL_MAX_CONNECTIONS + - name: backendPgsql.primary.extraEnvVars[0].value + value: ' ${BACKEND_POSTGRESQL_MAX_CONNECTIONS:-300}' - name: 'backendServer.schedulerEnabled' value: '$SCHEDULER_ENABLED' - name: 'backendServer.vaultConfirmWithdrawals' From db53aae81066ffeb51e32dfe068b0d73fed614cd Mon Sep 17 00:00:00 2001 From: Kacper Garbacinski <57113816+kgarbacinski@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:26:47 +0100 Subject: [PATCH 4/6] OCT-2248: Migrate logic for projects (#615) ## Description ## Definition of Done 1. [ ] If required, the desciption of your change is added to the [QA changelog](https://www.notion.so/octantapp/Changelog-for-the-QA-d96fa3b411cf488bb1d8d9a598d88281) 2. [ ] Acceptance criteria are met. 3. [ ] PR is manually tested before the merge by developer(s). - [ ] Happy path is manually checked. 4. [ ] PR is manually tested by QA when their assistance is required (1). - [ ] Octant Areas & Test Cases are checked for impact and updated if required (2). 5. [ ] Unit tests are added unless there is a reason to omit them. 6. [ ] Automated tests are added when required. 7. [ ] The code is merged. 8. [ ] Tech documentation is added / updated, reviewed and approved (including mandatory approval by a code owner, should such exist for changed files). - [ ] BE: Swagger documentation is updated. 9. [ ] When required by QA: - [ ] Deployed to the relevant environment. - [ ] Passed system tests. --- (1) Developer(s) in coordination with QA decide whether it's required. For small tickets introducing small changes QA assistance is most probably not required. (2) [Octant Areas & Test Cases](https://docs.google.com/spreadsheets/d/1cRe6dxuKJV3a4ZskAwWEPvrFkQm6rEfyUCYwLTYw_Cc). --- .../modules/projects/details/controller.py | 6 +- backend/v2/allocations/schemas.py | 1 + backend/v2/allocations/socket.py | 4 +- backend/v2/core/enums.py | 4 + backend/v2/core/logic.py | 5 + backend/v2/main.py | 2 + backend/v2/projects/contracts.py | 2 +- backend/v2/projects/core.py | 41 ++++++++ backend/v2/projects/dependencies.py | 97 ++++++++++++++++--- backend/v2/projects/repositories.py | 15 +++ backend/v2/projects/router.py | 42 ++++++++ backend/v2/projects/schemas.py | 35 +++++++ backend/v2/projects/services/__init__.py | 0 .../projects_allocation_threshold_getter.py} | 0 .../v2/projects/services/projects_details.py | 89 +++++++++++++++++ 15 files changed, 323 insertions(+), 20 deletions(-) create mode 100644 backend/v2/core/enums.py create mode 100644 backend/v2/core/logic.py create mode 100644 backend/v2/projects/core.py create mode 100644 backend/v2/projects/repositories.py create mode 100644 backend/v2/projects/router.py create mode 100644 backend/v2/projects/schemas.py create mode 100644 backend/v2/projects/services/__init__.py rename backend/v2/projects/{services.py => services/projects_allocation_threshold_getter.py} (100%) create mode 100644 backend/v2/projects/services/projects_details.py diff --git a/backend/app/modules/projects/details/controller.py b/backend/app/modules/projects/details/controller.py index ffb7ba1eee..28b2346e18 100644 --- a/backend/app/modules/projects/details/controller.py +++ b/backend/app/modules/projects/details/controller.py @@ -1,4 +1,4 @@ -from typing import List, Dict +from typing import List from app.context.manager import epoch_context from app.modules.registry import get_services @@ -7,7 +7,7 @@ def get_projects_details_for_multiple_params( epochs: List[int], search_phrases: List[str] -) -> List[Dict[str, str]]: +) -> list[dict[str, str]]: searched_projects = [] for epoch in epochs: for search_phrase in search_phrases: @@ -16,7 +16,7 @@ def get_projects_details_for_multiple_params( return searched_projects -def get_projects_details(epoch: int, search_phrase: str) -> List[Dict[str, str]]: +def get_projects_details(epoch: int, search_phrase: str) -> list[dict[str, str]]: context = epoch_context(epoch) service = get_services(context.epoch_state).projects_details_service diff --git a/backend/v2/allocations/schemas.py b/backend/v2/allocations/schemas.py index b3be442d9f..633059b196 100644 --- a/backend/v2/allocations/schemas.py +++ b/backend/v2/allocations/schemas.py @@ -1,6 +1,7 @@ from decimal import Decimal from pydantic import Field + from v2.core.types import Address, BigInteger, OctantModel diff --git a/backend/v2/allocations/socket.py b/backend/v2/allocations/socket.py index d4faa520f9..fee236a31f 100644 --- a/backend/v2/allocations/socket.py +++ b/backend/v2/allocations/socket.py @@ -37,7 +37,9 @@ get_projects_contracts, get_projects_settings, ) -from v2.projects.services import ProjectsAllocationThresholdGetter +from v2.projects.services.projects_allocation_threshold_getter import ( + ProjectsAllocationThresholdGetter, +) from v2.uniqueness_quotients.dependencies import ( get_uq_score_getter, get_uq_score_settings, diff --git a/backend/v2/core/enums.py b/backend/v2/core/enums.py new file mode 100644 index 0000000000..0115fc645a --- /dev/null +++ b/backend/v2/core/enums.py @@ -0,0 +1,4 @@ +class ChainTypes: + MAINNET = 1 + LOCAL = 1337 + SEPOLIA = 11155111 diff --git a/backend/v2/core/logic.py b/backend/v2/core/logic.py new file mode 100644 index 0000000000..60a79481eb --- /dev/null +++ b/backend/v2/core/logic.py @@ -0,0 +1,5 @@ +from v2.core.enums import ChainTypes + + +def compare_blockchain_types(chain_id: int, expected_chain: ChainTypes) -> bool: + return chain_id == expected_chain diff --git a/backend/v2/main.py b/backend/v2/main.py index 3d2f9bfcb5..006ef9985a 100644 --- a/backend/v2/main.py +++ b/backend/v2/main.py @@ -8,6 +8,7 @@ from fastapi.responses import JSONResponse from sqlalchemy.exc import SQLAlchemyError from v2.allocations.router import api as allocations_api +from v2.projects.router import api as projects_api from v2.allocations.socket import AllocateNamespace from v2.core.dependencies import get_socketio_settings from v2.project_rewards.router import api as project_rewards_api @@ -67,3 +68,4 @@ def get_socketio_manager() -> socketio.AsyncRedisManager | None: app.include_router(allocations_api) app.include_router(project_rewards_api) app.include_router(epochs_api) +app.include_router(projects_api) diff --git a/backend/v2/projects/contracts.py b/backend/v2/projects/contracts.py index e2ae34df55..8656b1a7b1 100644 --- a/backend/v2/projects/contracts.py +++ b/backend/v2/projects/contracts.py @@ -10,7 +10,7 @@ async def get_project_addresses(self, epoch_number: int) -> list[str]: ) return await self.contract.functions.getProposalAddresses(epoch_number).call() - async def get_project_cid(self): + async def get_project_cid(self) -> str: logging.debug("[Projects contract] Getting projects CID") return await self.contract.functions.cid().call() diff --git a/backend/v2/projects/core.py b/backend/v2/projects/core.py new file mode 100644 index 0000000000..a4222f0b16 --- /dev/null +++ b/backend/v2/projects/core.py @@ -0,0 +1,41 @@ +from app.infrastructure.database.models import ProjectsDetails + + +def parse_cids_to_epochs_dict(cids: list[str]) -> dict[int, str]: + """ + Convert a list of CIDs to a dictionary mapping epochs to CIDs. + """ + return {index: cid.strip() for index, cid in enumerate(cids, start=1)} + + +def process_search_params( + epochs: str, + search_phrases: str = "", +) -> tuple[list[int], list[str]]: + """ + Process and validate the input parameters. + Returns a tuple of (epoch_list, search_phrases_list) + """ + epoch_list = [int(e) for e in epochs.split(",")] + search_phrases_list = search_phrases.split(",") if search_phrases else [""] + return epoch_list, search_phrases_list + + +def filter_projects_details( + projects_details: list[ProjectsDetails], search_phrase: str +) -> list[ProjectsDetails]: + """ + Filter projects details by search phrase. + """ + search_phrase = search_phrase.strip().lower() + + filtered_project_details = [] + + for project_details in projects_details: + if ( + search_phrase in project_details.name.lower() + or search_phrase in project_details.address.lower() + ): + filtered_project_details.append(project_details) + + return filtered_project_details diff --git a/backend/v2/projects/dependencies.py b/backend/v2/projects/dependencies.py index 3cf6bfef4e..d090db1019 100644 --- a/backend/v2/projects/dependencies.py +++ b/backend/v2/projects/dependencies.py @@ -1,17 +1,50 @@ +from __future__ import annotations + from typing import Annotated from fastapi import Depends from pydantic import Field + +from app.constants import DEFAULT_MAINNET_PROJECT_CIDS from v2.core.dependencies import GetSession, OctantSettings, Web3 +from v2.core.enums import ChainTypes +from v2.core.logic import compare_blockchain_types from v2.epochs.dependencies import GetOpenAllocationWindowEpochNumber from v2.projects.contracts import PROJECTS_ABI, ProjectsContracts -from v2.projects.services import ProjectsAllocationThresholdGetter +from v2.projects.core import process_search_params +from v2.projects.schemas import EpochsParameter, SearchPhrasesParameter, EpochNumberPath +from v2.projects.services.projects_allocation_threshold_getter import ( + ProjectsAllocationThresholdGetter, +) +from v2.projects.services.projects_details import ( + ProjectsDetailsGetter, + ProjectsMetadataGetter, +) class ProjectsSettings(OctantSettings): projects_contract_address: str = Field( validation_alias="proposals_contract_address" ) + is_mainnet: bool = Field( + default_factory=lambda: compare_blockchain_types( + Field(validation_alias="chain_id"), ChainTypes.MAINNET + ) + ) + mainnet_project_cids_raw: str = Field( + validation_alias="mainnet_proposal_cids", default=DEFAULT_MAINNET_PROJECT_CIDS + ) + + @property + def mainnet_project_cids(self) -> list[str]: + return self.mainnet_project_cids_raw.split(",") + + +class ProjectsAllocationThresholdSettings(OctantSettings): + project_count_multiplier: int = Field( + default=1, + description="The multiplier to the number of projects to calculate the allocation threshold.", + ) def get_projects_settings() -> ProjectsSettings: @@ -21,20 +54,9 @@ def get_projects_settings() -> ProjectsSettings: def get_projects_contracts( w3: Web3, settings: Annotated[ProjectsSettings, Depends(get_projects_settings)] ) -> ProjectsContracts: - return ProjectsContracts(w3, PROJECTS_ABI, settings.projects_contract_address) # type: ignore[arg-type] - - -GetProjectsContracts = Annotated[ - ProjectsContracts, - Depends(get_projects_contracts), -] - - -class ProjectsAllocationThresholdSettings(OctantSettings): - project_count_multiplier: int = Field( - default=1, - description="The multiplier to the number of projects to calculate the allocation threshold.", - ) + return ProjectsContracts( + w3, PROJECTS_ABI, settings.projects_contract_address + ) # type: ignore[arg-type] def get_projects_allocation_threshold_settings() -> ProjectsAllocationThresholdSettings: @@ -53,3 +75,48 @@ def get_projects_allocation_threshold_getter( return ProjectsAllocationThresholdGetter( epoch_number, session, projects, settings.project_count_multiplier ) + + +async def get_projects_metadata_getter( + epoch_number: EpochNumberPath, + projects_contracts: GetProjectsContracts, + settings: GetProjectsSettings, + session: GetSession, +) -> ProjectsMetadataGetter: + return ProjectsMetadataGetter( + session=session, + epoch_number=epoch_number, + projects_contracts=projects_contracts, + is_mainnet=settings.is_mainnet, + mainnet_project_cids=settings.mainnet_project_cids, + ) + + +async def get_projects_details_getter( + session: GetSession, epochs: EpochsParameter, search_phrases: SearchPhrasesParameter +) -> ProjectsDetailsGetter: + epoch_numbers, search_phrases = process_search_params(epochs, search_phrases) + return ProjectsDetailsGetter( + session=session, epoch_numbers=epoch_numbers, search_phrases=search_phrases + ) + + +GetProjectsContracts = Annotated[ + ProjectsContracts, + Depends(get_projects_contracts), +] + +GetProjectsSettings = Annotated[ + ProjectsSettings, + Depends(get_projects_settings), +] + +GetProjectsMetadataGetter = Annotated[ + ProjectsMetadataGetter, + Depends(get_projects_metadata_getter), +] + +GetProjectsDetailsGetter = Annotated[ + ProjectsDetailsGetter, + Depends(get_projects_details_getter), +] diff --git a/backend/v2/projects/repositories.py b/backend/v2/projects/repositories.py new file mode 100644 index 0000000000..575a3ef599 --- /dev/null +++ b/backend/v2/projects/repositories.py @@ -0,0 +1,15 @@ +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.infrastructure.database.models import ProjectsDetails + + +async def get_projects_details_for_epoch( + session: AsyncSession, epoch: int +) -> list[ProjectsDetails]: + """Get project details for a specific epoch.""" + + result = await session.scalars( + select(ProjectsDetails).filter(ProjectsDetails.epoch == epoch) + ) + return list(result.all()) diff --git a/backend/v2/projects/router.py b/backend/v2/projects/router.py new file mode 100644 index 0000000000..7b4b4cdd5a --- /dev/null +++ b/backend/v2/projects/router.py @@ -0,0 +1,42 @@ +from fastapi import APIRouter + +from v2.projects.dependencies import ( + GetProjectsDetailsGetter, + GetProjectsMetadataGetter, +) +from v2.projects.schemas import ( + ProjectsMetadataResponseV1, + ProjectsDetailsResponseV1, +) + +api = APIRouter(prefix="/projects", tags=["Projects"]) + + +@api.get("/details", response_model=ProjectsDetailsResponseV1) +async def get_projects_details_v1( + projects_details_getter: GetProjectsDetailsGetter, +) -> ProjectsDetailsResponseV1: + """ + Returns projects details for given epochs and search phrases. + + Args: + epochs (str): Comma-separated list of epoch numbers. + search_phrases (str): Comma-separated search phrases to filter projects. + """ + projects_details = await projects_details_getter.get_by_search_phrase() + + return ProjectsDetailsResponseV1(projects_details=projects_details) + + +@api.get("/epoch/{epoch_number}", response_model=ProjectsMetadataResponseV1) +async def get_projects_metadata_v1( + projects_metadata_getter: GetProjectsMetadataGetter, +) -> ProjectsMetadataResponseV1: + """ + Returns projects metadata for a given epoch: addresses and CID. + + Args: + epoch (int): The epoch number to fetch metadata for. + """ + projects_metadata_response = await projects_metadata_getter.get() + return projects_metadata_response diff --git a/backend/v2/projects/schemas.py b/backend/v2/projects/schemas.py new file mode 100644 index 0000000000..4c32ff6ef2 --- /dev/null +++ b/backend/v2/projects/schemas.py @@ -0,0 +1,35 @@ +from typing import Annotated + +from fastapi import Query, Path +from pydantic import field_validator + +from v2.core.types import OctantModel + + +class ProjectModel(OctantModel): + name: str + address: str + epoch: str + + @field_validator("epoch", mode="before") + def convert_epoch_to_str(cls, value): + """Ensure epoch is always a string.""" + return str(value) + + +class ProjectsMetadataResponseV1(OctantModel): + projects_addresses: list[str] + projects_cid: str + + +class ProjectsDetailsResponseV1(OctantModel): + projects_details: list[ProjectModel] + + +EpochsParameter = Annotated[ + str, Query(..., description="Comma-separated list of epoch numbers") +] +SearchPhrasesParameter = Annotated[ + str, Query(..., alias="searchPhrases", description="Comma-separated search phrases") +] +EpochNumberPath = Annotated[int, Path(..., description="Epoch number")] diff --git a/backend/v2/projects/services/__init__.py b/backend/v2/projects/services/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/v2/projects/services.py b/backend/v2/projects/services/projects_allocation_threshold_getter.py similarity index 100% rename from backend/v2/projects/services.py rename to backend/v2/projects/services/projects_allocation_threshold_getter.py diff --git a/backend/v2/projects/services/projects_details.py b/backend/v2/projects/services/projects_details.py new file mode 100644 index 0000000000..fa8453656c --- /dev/null +++ b/backend/v2/projects/services/projects_details.py @@ -0,0 +1,89 @@ +import logging +from dataclasses import dataclass + +from sqlalchemy.ext.asyncio import AsyncSession + +from v2.projects.contracts import ProjectsContracts +from v2.projects.core import filter_projects_details, parse_cids_to_epochs_dict +from v2.projects.repositories import get_projects_details_for_epoch +from v2.projects.schemas import ProjectModel, ProjectsMetadataResponseV1 + + +@dataclass +class ProjectsDetailsGetter: + # Parameters + epoch_numbers: list[int] + search_phrases: list[str] + + # Dependencies + session: AsyncSession + + async def get_by_search_phrase(self) -> list[ProjectModel]: + """ + Get projects details filtered by search phrase for a specific epoch context. + """ + + all_filtered_projects_details = [] + + for epoch_number in self.epoch_numbers: + projects_details = await get_projects_details_for_epoch( + self.session, epoch_number + ) + for search_phrase in self.search_phrases: + filtered_projects_details = filter_projects_details( + projects_details, search_phrase + ) + + for project_details in filtered_projects_details: + project_details_model = ProjectModel( + name=project_details.name, + address=project_details.address, + epoch=project_details.epoch, + ) + all_filtered_projects_details.append(project_details_model) + + return all_filtered_projects_details + + +@dataclass +class ProjectsMetadataGetter: + # Parameters + epoch_number: int + is_mainnet: bool + mainnet_project_cids: list[str] + + # Dependencies + session: AsyncSession + projects_contracts: ProjectsContracts + + async def _get_projects_cid(self) -> str: + if self.is_mainnet: + epoch_to_cid_dict = parse_cids_to_epochs_dict(self.mainnet_project_cids) + projects_cid = ( + await self.projects_contracts.get_project_cid() + if self.epoch_number not in epoch_to_cid_dict + else epoch_to_cid_dict[self.epoch_number] + ) + else: + projects_cid = await self.projects_contracts.get_project_cid() + + return projects_cid + + async def get(self) -> ProjectsMetadataResponseV1: + """ + Get projects metadata for a specific epoch. + """ + logging.debug(f"Getting projects metadata for epoch {self.epoch_number}") + + projects_cid = await self._get_projects_cid() + + logging.debug(f"Projects CID: {projects_cid}") + + projects_address_list = await self.projects_contracts.get_project_addresses( + self.epoch_number + ) + logging.debug(f"Projects Address List: {projects_address_list}") + + return ProjectsMetadataResponseV1( + projects_addresses=projects_address_list, projects_cid=projects_cid + ) From 1f12360aebf70ce73ed68a1a0767c73a37efc4be Mon Sep 17 00:00:00 2001 From: Kacper Garbacinski <57113816+kgarbacinski@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:40:21 +0100 Subject: [PATCH 5/6] OCT-2249: Add unit tests for /projects & improve searching (#627) ## Description ## Definition of Done 1. [ ] If required, the desciption of your change is added to the [QA changelog](https://www.notion.so/octantapp/Changelog-for-the-QA-d96fa3b411cf488bb1d8d9a598d88281) 2. [ ] Acceptance criteria are met. 3. [ ] PR is manually tested before the merge by developer(s). - [ ] Happy path is manually checked. 4. [ ] PR is manually tested by QA when their assistance is required (1). - [ ] Octant Areas & Test Cases are checked for impact and updated if required (2). 5. [ ] Unit tests are added unless there is a reason to omit them. 6. [ ] Automated tests are added when required. 7. [ ] The code is merged. 8. [ ] Tech documentation is added / updated, reviewed and approved (including mandatory approval by a code owner, should such exist for changed files). - [ ] BE: Swagger documentation is updated. 9. [ ] When required by QA: - [ ] Deployed to the relevant environment. - [ ] Passed system tests. --- (1) Developer(s) in coordination with QA decide whether it's required. For small tickets introducing small changes QA assistance is most probably not required. (2) [Octant Areas & Test Cases](https://docs.google.com/spreadsheets/d/1cRe6dxuKJV3a4ZskAwWEPvrFkQm6rEfyUCYwLTYw_Cc). --- backend/app/constants.py | 1 + backend/tests/v2/factories/__init__.py | 3 + .../tests/v2/factories/projects_details.py | 40 +++++ backend/tests/v2/fake_contracts/conftest.py | 26 +++- backend/tests/v2/fake_contracts/helpers.py | 14 ++ backend/tests/v2/fake_contracts/projects.py | 23 +++ backend/tests/v2/projects/conftest.py | 52 +++++++ .../v2/projects/test_projects_details.py | 144 ++++++++++++++++++ .../v2/projects/test_projects_metadata.py | 99 ++++++++++++ backend/v2/projects/dependencies.py | 8 +- backend/v2/projects/schemas.py | 2 + .../v2/projects/services/projects_details.py | 4 + 12 files changed, 413 insertions(+), 3 deletions(-) create mode 100644 backend/tests/v2/factories/projects_details.py create mode 100644 backend/tests/v2/fake_contracts/projects.py create mode 100644 backend/tests/v2/projects/conftest.py create mode 100644 backend/tests/v2/projects/test_projects_details.py create mode 100644 backend/tests/v2/projects/test_projects_metadata.py diff --git a/backend/app/constants.py b/backend/app/constants.py index 40dc892695..bd98b13a23 100644 --- a/backend/app/constants.py +++ b/backend/app/constants.py @@ -26,6 +26,7 @@ SAFE_API_SEPOLIA = "https://safe-transaction-sepolia.safe.global/api/v1" DEFAULT_MAINNET_PROJECT_CIDS = "QmSQEFD35gKxdPEmngNt1CWe3kSwiiGqBn1Z3FZvWb8mvK,Qmds9N5y2vkMuPTD6M4EBxNXnf3bjTDmzWBGnCkQGsMMGe,QmSXcT18anMXKACTueom8GXw8zrxTBbHGB71atitf6gZ9V,QmXomSdCCwt4FtBp3pidqSz3PtaiV2EyQikU6zRGWeCAsf,QmdtFLK3sB7EwQTNaqtmBnZqnN2pYZcu6GmUSTrpvb9wcq" +DEFAULT_PROJECTS_CONTRACT_ADDRESS = "0xB259fe6EC190cffF893b247AE688eFBF4261D2fc" EPOCH0_SYBILS = [ "0xde19a6ce83cc934e5d4c4573f0f026c02c984fb2", diff --git a/backend/tests/v2/factories/__init__.py b/backend/tests/v2/factories/__init__.py index 257d44eb9a..561003f34f 100644 --- a/backend/tests/v2/factories/__init__.py +++ b/backend/tests/v2/factories/__init__.py @@ -2,6 +2,7 @@ from tests.v2.factories.allocation_requests import AllocationRequestFactorySet from tests.v2.factories.allocations import AllocationFactorySet +from tests.v2.factories.projects_details import ProjectsDetailsFactorySet from tests.v2.factories.users import UserFactorySet from dataclasses import dataclass @@ -21,6 +22,7 @@ class FactoriesAggregator: users: UserFactorySet allocation_requests: AllocationRequestFactorySet allocations: AllocationFactorySet + projects_details: ProjectsDetailsFactorySet def __init__(self, fast_session: AsyncSession): """ @@ -29,3 +31,4 @@ def __init__(self, fast_session: AsyncSession): self.users = UserFactorySet(fast_session) self.allocation_requests = AllocationRequestFactorySet(fast_session) self.allocations = AllocationFactorySet(fast_session) + self.projects_details = ProjectsDetailsFactorySet(fast_session) diff --git a/backend/tests/v2/factories/projects_details.py b/backend/tests/v2/factories/projects_details.py new file mode 100644 index 0000000000..0abf339eaa --- /dev/null +++ b/backend/tests/v2/factories/projects_details.py @@ -0,0 +1,40 @@ +from async_factory_boy.factory.sqlalchemy import AsyncSQLAlchemyFactory +from factory import LazyAttribute + +from app.infrastructure.database.models import ProjectsDetails +from tests.v2.factories.base import FactorySetBase +from tests.v2.factories.helpers import generate_random_eip55_address +from v2.core.types import Address + + +class ProjectsDetailsFactory(AsyncSQLAlchemyFactory): + class Meta: + model = ProjectsDetails + sqlalchemy_session_persistence = "commit" + + address = LazyAttribute(lambda _: generate_random_eip55_address()) + name = None + epoch = None + + +class ProjectsDetailsFactorySet(FactorySetBase): + _factories = {"projects_details": ProjectsDetailsFactory} + + async def create( + self, + name: str, + epoch: int, + address: Address | None = None, + ) -> ProjectsDetails: + factory_kwargs = { + "address": address, + "name": name, + "epoch": epoch, + } + + if address is not None: + factory_kwargs["address"] = address + + projects_details = await ProjectsDetailsFactory.create(**factory_kwargs) + + return projects_details diff --git a/backend/tests/v2/fake_contracts/conftest.py b/backend/tests/v2/fake_contracts/conftest.py index fda141cf12..d5881ab123 100644 --- a/backend/tests/v2/fake_contracts/conftest.py +++ b/backend/tests/v2/fake_contracts/conftest.py @@ -6,8 +6,13 @@ from fastapi import FastAPI from tests.v2.fake_contracts.epochs import FakeEpochsContract -from tests.v2.fake_contracts.helpers import FakeEpochsContractDetails +from tests.v2.fake_contracts.helpers import ( + FakeEpochsContractDetails, + FakeProjectsContractDetails, +) +from tests.v2.fake_contracts.projects import FakeProjectsContract from v2.epochs.dependencies import get_epochs_contracts +from v2.projects.dependencies import get_projects_contracts @pytest.fixture(scope="function") @@ -26,6 +31,25 @@ def _create_fake_epochs_contract( return _create_fake_epochs_contract +@pytest.fixture(scope="function") +def fake_projects_contract_factory(fast_app: FastAPI) -> FakeProjectsContractCallable: + def _create_fake_projects_contract( + projects_details_for_contract: FakeProjectsContractDetails | None = None, + ): + fake_projects_contract = FakeProjectsContract(projects_details_for_contract) + + fast_app.dependency_overrides[ + get_projects_contracts + ] = lambda: fake_projects_contract + + return fake_projects_contract + + return _create_fake_projects_contract + + FakeEpochsContractCallable = Callable[ [FakeEpochsContractDetails | None], FakeEpochsContract ] +FakeProjectsContractCallable = Callable[ + [FakeProjectsContractDetails | None], FakeProjectsContract +] diff --git a/backend/tests/v2/fake_contracts/helpers.py b/backend/tests/v2/fake_contracts/helpers.py index 933f71d096..18f0d61f05 100644 --- a/backend/tests/v2/fake_contracts/helpers.py +++ b/backend/tests/v2/fake_contracts/helpers.py @@ -1,5 +1,7 @@ from dataclasses import dataclass +from v2.core.types import Address + @dataclass class FakeEpochsContractDetails: @@ -13,3 +15,15 @@ class FakeEpochsContractDetails: future_epoch_props: dict = None is_started: bool = False started: int = 0 + + +@dataclass +class FakeProjectDetails: + address: Address + epoch_number: int + + +@dataclass +class FakeProjectsContractDetails: + projects_cid: str + projects_details: list[FakeProjectDetails] diff --git a/backend/tests/v2/fake_contracts/projects.py b/backend/tests/v2/fake_contracts/projects.py new file mode 100644 index 0000000000..247c63645b --- /dev/null +++ b/backend/tests/v2/fake_contracts/projects.py @@ -0,0 +1,23 @@ +from tests.v2.fake_contracts.helpers import FakeProjectsContractDetails + + +class FakeProjectsContract: + def __init__(self, projects_details_for_contract: FakeProjectsContractDetails): + self.projects_details_for_contract = projects_details_for_contract + + async def get_project_addresses(self, epoch_number: int) -> list[str]: + filtered_projects_details = list( + filter( + lambda project_details: project_details.epoch_number == epoch_number, + self.projects_details_for_contract.projects_details, + ) + ) + return list( + map( + lambda project_details: project_details.address, + filtered_projects_details, + ) + ) + + async def get_project_cid(self) -> str: + return self.projects_details_for_contract.projects_cid diff --git a/backend/tests/v2/projects/conftest.py b/backend/tests/v2/projects/conftest.py new file mode 100644 index 0000000000..cd292b1b80 --- /dev/null +++ b/backend/tests/v2/projects/conftest.py @@ -0,0 +1,52 @@ +import pytest + +from tests.v2.fake_contracts.conftest import ( # noqa: F401 + fake_projects_contract_factory, +) + +from tests.v2.factories.helpers import generate_random_eip55_address + + +@pytest.fixture(scope="module") +def multiple_project_details_simple() -> list: + details = [ + { + "name": "test_project1", + "epoch": 3, + "address": generate_random_eip55_address(), + }, + { + "name": "test_project2", + "epoch": 2, + "address": generate_random_eip55_address(), + }, + { + "name": "test_project3", + "epoch": 1, + "address": generate_random_eip55_address(), + }, + ] + + return details + + +@pytest.fixture(scope="module") +def multiple_project_details_various() -> list: + details = [ + { + "name": "test_project1", + "epoch": 3, + "address": generate_random_eip55_address(), + }, + { + "name": "octant_project2", + "epoch": 2, + "address": generate_random_eip55_address(), + }, + { + "name": "great_project3", + "epoch": 1, + "address": generate_random_eip55_address(), + }, + ] + return details diff --git a/backend/tests/v2/projects/test_projects_details.py b/backend/tests/v2/projects/test_projects_details.py new file mode 100644 index 0000000000..941b090fdc --- /dev/null +++ b/backend/tests/v2/projects/test_projects_details.py @@ -0,0 +1,144 @@ +from http import HTTPStatus + +import pytest +from httpx import AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession + +from tests.v2.factories import FactoriesAggregator +from tests.v2.factories.helpers import generate_random_eip55_address + + +@pytest.mark.asyncio +async def test_returns_empty_project_details( + fast_client: AsyncClient, fast_session: AsyncSession +): + async with fast_client as client: + resp = await client.get("projects/details?epochs=1&searchPhrases=test") + assert resp.status_code == HTTPStatus.OK + assert resp.json() == {"projectsDetails": []} + + +@pytest.mark.asyncio +async def test_returns_correct_project_details_for_single_filter( + fast_client: AsyncClient, factories: FactoriesAggregator +): + single_details = { + "name": "test_project1", + "epoch": 3, + "address": generate_random_eip55_address(), + } + + await factories.projects_details.create(**single_details) + + async with fast_client as client: + resp = await client.get("projects/details?epochs=3&searchPhrases=test_project") + assert resp.status_code == HTTPStatus.OK + assert resp.json() == { + "projectsDetails": [ + { + "name": single_details["name"], + "epoch": str(single_details["epoch"]), + "address": single_details["address"], + } + ] + } + + +@pytest.mark.asyncio +async def test_returns_correct_project_details_for_many_epochs( + fast_client: AsyncClient, + factories: FactoriesAggregator, + multiple_project_details_simple: list, +): + details = multiple_project_details_simple + + for detail in details: + await factories.projects_details.create(**detail) + + async with fast_client as client: + resp = await client.get("projects/details?epochs=1,2,3&searchPhrases=test") + assert resp.status_code == HTTPStatus.OK + + assert resp.json() == { + "projectsDetails": [ + { + "name": details[2]["name"], + "epoch": str(details[2]["epoch"]), + "address": details[2]["address"], + }, + { + "name": details[1]["name"], + "epoch": str(details[1]["epoch"]), + "address": details[1]["address"], + }, + { + "name": details[0]["name"], + "epoch": str(details[0]["epoch"]), + "address": details[0]["address"], + }, + ] + } + + +@pytest.mark.asyncio +async def test_returns_correct_project_details_for_many_epochs_and_search_phrases( + fast_client: AsyncClient, + factories: FactoriesAggregator, + multiple_project_details_various: list, +): + details = multiple_project_details_various + + for detail in details: + await factories.projects_details.create(**detail) + + async with fast_client as client: + resp = await client.get( + "projects/details?epochs=1,2,3&searchPhrases=test,octant,great" + ) + assert resp.status_code == HTTPStatus.OK + + assert resp.json() == { + "projectsDetails": [ + { + "name": details[2]["name"], + "epoch": str(details[2]["epoch"]), + "address": details[2]["address"], + }, + { + "name": details[1]["name"], + "epoch": str(details[1]["epoch"]), + "address": details[1]["address"], + }, + { + "name": details[0]["name"], + "epoch": str(details[0]["epoch"]), + "address": details[0]["address"], + }, + ] + } + + +@pytest.mark.asyncio +async def test_returns_not_duplicated_project_details_for_words_from_the_same_project( + fast_client: AsyncClient, + factories: FactoriesAggregator, + multiple_project_details_various: list, +): + details = multiple_project_details_various + words = "test,project1" + + for detail in details: + await factories.projects_details.create(**detail) + + async with fast_client as client: + resp = await client.get(f"projects/details?epochs=3&searchPhrases={words}") + assert resp.status_code == HTTPStatus.OK + assert resp.json() == { + "projectsDetails": [ + { + "name": details[0]["name"], + "epoch": str(details[0]["epoch"]), + "address": details[0]["address"], + } + ] + } diff --git a/backend/tests/v2/projects/test_projects_metadata.py b/backend/tests/v2/projects/test_projects_metadata.py new file mode 100644 index 0000000000..daf56e4218 --- /dev/null +++ b/backend/tests/v2/projects/test_projects_metadata.py @@ -0,0 +1,99 @@ +from http import HTTPStatus + +import pytest +from httpx import AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession + +from tests.v2.factories import FactoriesAggregator +from tests.v2.factories.helpers import generate_random_eip55_address +from tests.v2.fake_contracts.conftest import FakeProjectsContractCallable +from tests.v2.fake_contracts.helpers import ( + FakeProjectsContractDetails, + FakeProjectDetails, +) + + +@pytest.mark.asyncio +async def test_returns_correct_projects_metadata_for_epoch( + fake_projects_contract_factory: FakeProjectsContractCallable, + fast_client: AsyncClient, + fast_session: AsyncSession, +): + epoch_number = 3 + projects_cid = "Qm123456789abcdef" + project_details = [ + FakeProjectDetails( + address=generate_random_eip55_address(), epoch_number=epoch_number + ) + ] + + fake_projects_contract_details = FakeProjectsContractDetails( + projects_cid=projects_cid, projects_details=project_details + ) + + fake_projects_contract_factory(fake_projects_contract_details) + + async with fast_client as client: + resp = await client.get(f"projects/epoch/{epoch_number}") + assert resp.status_code == HTTPStatus.OK + assert resp.json() == { + "projectsAddresses": [project_details[0].address], + "projectsCid": projects_cid, + } + + +@pytest.mark.asyncio +async def test_returns_multiple_projects_metadata_for_epoch( + fake_projects_contract_factory: FakeProjectsContractCallable, + fast_client: AsyncClient, + factories: FactoriesAggregator, +): + epoch_number = 3 + projects_cid = "Qm123456789abcdef" + project_details = [ + FakeProjectDetails( + address=generate_random_eip55_address(), epoch_number=epoch_number + ) + for _ in range(3) + ] + + fake_projects_contract_details = FakeProjectsContractDetails( + projects_cid=projects_cid, projects_details=project_details + ) + + fake_projects_contract_factory(fake_projects_contract_details) + + async with fast_client as client: + resp = await client.get(f"projects/epoch/{epoch_number}") + assert resp.status_code == HTTPStatus.OK + assert resp.json() == { + "projectsAddresses": [detail.address for detail in project_details], + "projectsCid": projects_cid, + } + + +@pytest.mark.asyncio +async def test_returns_empty_list_of_projects_addresses_for_nonexistent_epoch( + fake_projects_contract_factory: FakeProjectsContractCallable, + fast_client: AsyncClient, + factories: FactoriesAggregator, +): + epoch_number = 3 + non_existent_epoch = 4 + + projects_cid = "Qm123456789abcdef" + fake_projects_contract_details = FakeProjectsContractDetails( + projects_cid=projects_cid, + projects_details=[ + FakeProjectDetails( + address=generate_random_eip55_address(), epoch_number=epoch_number + ) + ], + ) + + fake_projects_contract_factory(fake_projects_contract_details) + + async with fast_client as client: + resp = await client.get(f"projects/epoch/{non_existent_epoch}") + assert resp.status_code == HTTPStatus.OK + assert resp.json() == {"projectsAddresses": [], "projectsCid": projects_cid} diff --git a/backend/v2/projects/dependencies.py b/backend/v2/projects/dependencies.py index d090db1019..116c919007 100644 --- a/backend/v2/projects/dependencies.py +++ b/backend/v2/projects/dependencies.py @@ -5,7 +5,10 @@ from fastapi import Depends from pydantic import Field -from app.constants import DEFAULT_MAINNET_PROJECT_CIDS +from app.constants import ( + DEFAULT_MAINNET_PROJECT_CIDS, + DEFAULT_PROJECTS_CONTRACT_ADDRESS, +) from v2.core.dependencies import GetSession, OctantSettings, Web3 from v2.core.enums import ChainTypes from v2.core.logic import compare_blockchain_types @@ -24,7 +27,8 @@ class ProjectsSettings(OctantSettings): projects_contract_address: str = Field( - validation_alias="proposals_contract_address" + validation_alias="proposals_contract_address", + default=DEFAULT_PROJECTS_CONTRACT_ADDRESS, ) is_mainnet: bool = Field( default_factory=lambda: compare_blockchain_types( diff --git a/backend/v2/projects/schemas.py b/backend/v2/projects/schemas.py index 4c32ff6ef2..18b995f8e5 100644 --- a/backend/v2/projects/schemas.py +++ b/backend/v2/projects/schemas.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Annotated from fastapi import Query, Path diff --git a/backend/v2/projects/services/projects_details.py b/backend/v2/projects/services/projects_details.py index fa8453656c..fc52a59b13 100644 --- a/backend/v2/projects/services/projects_details.py +++ b/backend/v2/projects/services/projects_details.py @@ -40,6 +40,10 @@ async def get_by_search_phrase(self) -> list[ProjectModel]: address=project_details.address, epoch=project_details.epoch, ) + + if project_details_model in all_filtered_projects_details: + continue + all_filtered_projects_details.append(project_details_model) return all_filtered_projects_details From 5d5d16dfe2d9c626aea4772542d3466a10432268 Mon Sep 17 00:00:00 2001 From: Kacper Garbacinski <57113816+kgarbacinski@users.noreply.github.com> Date: Wed, 15 Jan 2025 09:14:26 +0100 Subject: [PATCH 6/6] OCT-2296: Tweak defaults of PG_POOL when max_connections increased (#616) ## Description ## Definition of Done 1. [ ] If required, the desciption of your change is added to the [QA changelog](https://www.notion.so/octantapp/Changelog-for-the-QA-d96fa3b411cf488bb1d8d9a598d88281) 2. [ ] Acceptance criteria are met. 3. [ ] PR is manually tested before the merge by developer(s). - [ ] Happy path is manually checked. 4. [ ] PR is manually tested by QA when their assistance is required (1). - [ ] Octant Areas & Test Cases are checked for impact and updated if required (2). 5. [ ] Unit tests are added unless there is a reason to omit them. 6. [ ] Automated tests are added when required. 7. [ ] The code is merged. 8. [ ] Tech documentation is added / updated, reviewed and approved (including mandatory approval by a code owner, should such exist for changed files). - [ ] BE: Swagger documentation is updated. 9. [ ] When required by QA: - [ ] Deployed to the relevant environment. - [ ] Passed system tests. --- (1) Developer(s) in coordination with QA decide whether it's required. For small tickets introducing small changes QA assistance is most probably not required. (2) [Octant Areas & Test Cases](https://docs.google.com/spreadsheets/d/1cRe6dxuKJV3a4ZskAwWEPvrFkQm6rEfyUCYwLTYw_Cc). --- backend/v2/core/dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/v2/core/dependencies.py b/backend/v2/core/dependencies.py index ea9dbc514e..4c238dcf0f 100644 --- a/backend/v2/core/dependencies.py +++ b/backend/v2/core/dependencies.py @@ -33,13 +33,13 @@ def get_w3( class DatabaseSettings(OctantSettings): """ - Values below are the defaults for the database with max_connetions = 100 and backend pods = 3. + Values below are the defaults for the pod which can serve up to 100 connections. """ db_uri: str = Field(..., alias="db_uri") - pg_pool_size: int = Field(20, alias="sqlalchemy_connection_pool_size") - pg_max_overflow: int = Field(13, alias="sqlalchemy_connection_pool_max_overflow") + pg_pool_size: int = Field(67, alias="sqlalchemy_connection_pool_size") + pg_max_overflow: int = Field(33, alias="sqlalchemy_connection_pool_max_overflow") pg_pool_timeout: int = 60 pg_pool_recycle: int = 30 * 60 # 30 minutes pg_pool_pre_ping: bool = True