From 0847facf0bdc8f157bd83593d2c0e47887a16935 Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Tue, 13 Dec 2022 13:49:03 +0100 Subject: [PATCH] [V1] Add surplus_fee to slippage query (#156) Separating #155 into two "independent" PRs so we can add tests n stuff. The slippage query has been altered to account for surplus_fee. A new test is added, demonstrating that this surplus fee is captured by the V1 query and not the V2 query (since this is not yet possible). Note that while testing this, I have discovered a bug in our base slippage query. Receiver = 0 There are only 113 total trades over all time with this set (see here) which had to be fixed before we could properly test this. That is because the query thought that there was an AMM_OUT transfer. --- queries/dune_v1/period_slippage.sql | 11 ++- queries/dune_v2/period_slippage.sql | 7 +- src/queries.py | 4 +- src/update/reward_history.py | 4 +- tests/integration/common.py | 37 +++++++++ tests/integration/test_solver_slippage.py | 91 ++++++++++++++--------- 6 files changed, 111 insertions(+), 43 deletions(-) diff --git a/queries/dune_v1/period_slippage.sql b/queries/dune_v1/period_slippage.sql index d27c9fde..8cf4ea4b 100644 --- a/queries/dune_v1/period_slippage.sql +++ b/queries/dune_v1/period_slippage.sql @@ -8,14 +8,21 @@ filtered_trades as ( solver_name, solver_address, trader as trader_in, - receiver as trader_out, + -- Null Receiver: https://dune.com/queries/1729130 + case + when receiver = '\x0000000000000000000000000000000000000000' + then trader + else receiver end as trader_out, sell_token_address as "sellToken", buy_token_address as "buyToken", - atoms_sold as "sellAmount", + atoms_sold - coalesce(surplus_fee, 0) as "sellAmount", atoms_bought as "buyAmount", '\x9008D19f58AAbD9eD0D60971565AA8510560ab41' :: bytea as contract_address from gnosis_protocol_v2."trades" t join gnosis_protocol_v2."batches" b on t.tx_hash = b.tx_hash + left outer join dune_user_generated.cow_order_rewards_barn f + on f.tx_hash = t.tx_hash + and f.order_uid = t.order_uid where b.block_time between '{{StartTime}}' and '{{EndTime}}' and case diff --git a/queries/dune_v2/period_slippage.sql b/queries/dune_v2/period_slippage.sql index 10cbcff8..fd19f69f 100644 --- a/queries/dune_v2/period_slippage.sql +++ b/queries/dune_v2/period_slippage.sql @@ -11,7 +11,12 @@ filtered_trades as ( num_trades, solver_address, trader as trader_in, - receiver as trader_out, + -- Null Receiver: https://dune.com/queries/1729130 + -- TODO - This could also be fixed in the spellbook! + case + when receiver = '0x0000000000000000000000000000000000000000' + then trader + else receiver end as trader_out, sell_token_address as sell_token, buy_token_address as buy_token, atoms_sold, diff --git a/src/queries.py b/src/queries.py index 74104faf..d8503cee 100644 --- a/src/queries.py +++ b/src/queries.py @@ -95,7 +95,7 @@ def with_params( "PERIOD_SLIPPAGE": QueryData( name="Solver Slippage for Period", filepath="period_slippage.sql", - v1_id=1570227, - v2_id=1570561, + v1_id=1728478, + v2_id=1729274, ), } diff --git a/src/update/reward_history.py b/src/update/reward_history.py index 79cf0713..385fa25d 100644 --- a/src/update/reward_history.py +++ b/src/update/reward_history.py @@ -57,9 +57,7 @@ def __str__(self) -> str: solver, tx_hash, order_id = list( map(pg_hex2bytea, [self.solver, self.tx_hash, self.order_uid]) ) - return ( - f"('{order_id}','{solver}','{tx_hash}', {self.surplus_fee},{self.amount})" - ) + return f"('{order_id}','{solver}','{tx_hash}',{self.surplus_fee},{self.amount})" def fetch_and_push_order_rewards(dune: DuneAPI, env: Environment) -> None: diff --git a/tests/integration/common.py b/tests/integration/common.py index 85b0365b..e3a597b1 100644 --- a/tests/integration/common.py +++ b/tests/integration/common.py @@ -2,6 +2,10 @@ from dune_client.client import DuneClient from dune_client.query import Query +from dune_client.types import QueryParameter + +from src.models.accounting_period import AccountingPeriod +from src.queries import QUERIES def exec_or_get(dune: DuneClient, query: Query, result_id: Optional[str] = None): @@ -10,3 +14,36 @@ def exec_or_get(dune: DuneClient, query: Query, result_id: Optional[str] = None) print(f"Execution ID: {results.execution_id}") return results return dune.get_result(result_id) + + +def get_slippage_cte_rows( + dune: DuneClient, + cte_name: str, + period: AccountingPeriod, + tx_hash: Optional[str] = None, + v1_cache: Optional[str] = None, + v2_cache: Optional[str] = None, +): + slippage_query = QUERIES["PERIOD_SLIPPAGE"] + v1_query_id = slippage_query.v1_query.query_id + v2_query_id = slippage_query.v2_query.query_id + + parameters = period.as_query_params() + parameters.append(QueryParameter.enum_type("CTE_NAME", cte_name)) + if tx_hash: + parameters.append(QueryParameter.text_type("TxHash", tx_hash)) + + v1_results = exec_or_get( + dune, + Query(v1_query_id, params=parameters), + v1_cache, + ) + v2_results = exec_or_get( + dune, + Query(v2_query_id, params=parameters), + v2_cache, + ) + + v1_rows = v1_results.get_rows() + v2_rows = v2_results.get_rows() + return v1_rows, v2_rows diff --git a/tests/integration/test_solver_slippage.py b/tests/integration/test_solver_slippage.py index b74e4b7a..760ed73b 100644 --- a/tests/integration/test_solver_slippage.py +++ b/tests/integration/test_solver_slippage.py @@ -7,14 +7,12 @@ from dune_client.client import DuneClient from dune_client.file.interface import FileIO -from dune_client.query import Query -from dune_client.types import QueryParameter from src.constants import FILE_OUT_DIR from src.models.accounting_period import AccountingPeriod from src.models.slippage import SplitSlippages from src.queries import QUERIES -from tests.integration.common import exec_or_get +from tests.integration.common import get_slippage_cte_rows @dataclass @@ -75,33 +73,6 @@ def setUp(self) -> None: self.parameters = self.period.as_query_params() self.writer = FileIO(FILE_OUT_DIR) - def get_cte_rows( - self, - cte_name: str, - tx_hash: Optional[str] = None, - v1_cache: Optional[str] = None, - v2_cache: Optional[str] = None, - ): - parameters = self.period.as_query_params() - parameters.append(QueryParameter.enum_type("CTE_NAME", cte_name)) - if tx_hash: - parameters.append(QueryParameter.text_type("TxHash", tx_hash)) - - v1_results = exec_or_get( - self.dune, - Query(self.slippage_query.v1_query.query_id, params=parameters), - v1_cache, - ) - v2_results = exec_or_get( - self.dune, - Query(self.slippage_query.v2_query.query_id, params=parameters), - v2_cache, - ) - - v1_rows = v1_results.get_rows() - v2_rows = v2_results.get_rows() - return v1_rows, v2_rows - def get_cte_results( self, cte_name: str, @@ -109,15 +80,19 @@ def get_cte_results( v1_cache: Optional[str] = None, v2_cache: Optional[str] = None, ): - v1_rows, v2_rows = self.get_cte_rows(cte_name, tx_hash, v1_cache, v2_cache) + v1_rows, v2_rows = get_slippage_cte_rows( + self.dune, cte_name, self.period, tx_hash, v1_cache, v2_cache + ) self.writer.write_csv(v1_rows, f"{cte_name}_{tx_hash}_v1.csv") self.writer.write_csv(v2_rows, f"{cte_name}_{tx_hash}_v2.csv") return Comparison.from_dune_results(v1_rows, v2_rows) def test_batch_transfers(self): table_name = "batch_transfers" - data = self.get_cte_results( + data = get_slippage_cte_rows( + self.dune, table_name, + self.period, v1_cache="01GJQNSER0PYSMGHMEBDT03805", v2_cache="01GJQNSER0PYSMGHMEBDT03805", ) @@ -138,8 +113,10 @@ def test_batch_transfers(self): def test_incoming_and_outgoing(self): # This test demonstrates that records are "essentially" matching up to this table. table_name = "incoming_and_outgoing" - data = self.get_cte_results( + data = get_slippage_cte_rows( + self.dune, table_name, + self.period, v1_cache="01GJR1HKR37V7HRPTRJCDMZBAX", v2_cache="01GJR1HTNS87RKTV90WKH4TVSC", ) @@ -171,8 +148,10 @@ def test_incoming_and_outgoing(self): def test_final_token_balance_sheet(self): table_name = "final_token_balance_sheet" - data = self.get_cte_results( + data = get_slippage_cte_rows( + self.dune, table_name, + self.period, # Results for Period(2022-11-01) # v1_not_v2 = 172 batches # v2_not_v1 = 107 batches @@ -214,8 +193,10 @@ def test_final_token_balance_sheet(self): def test_similar_slippage_for_period(self): table_name = "results" - v1_result, v2_result = self.get_cte_rows( + v1_result, v2_result = get_slippage_cte_rows( + self.dune, table_name, + self.period, # --------------------------------------- # Results for Period(2022-11-01) # Negative: 0.0094 difference --> 9.4 USD @@ -257,6 +238,46 @@ def test_similar_slippage_for_period(self): delta=delta, # |v1 - v2| ~ 0.004 --> 4 USD (with ETH at 1000) ) + def test_limit_order_slippage(self): + """This is an internal buffer trade containing a limit order (with a surplus_fee)""" + table_name = "incoming_and_outgoing" + v1_result, v2_result = get_slippage_cte_rows( + self.dune, + table_name, + period=AccountingPeriod("2022-11-29", 1), + tx_hash="0xfe4589525c1ed764273fbca9120b0e5f7f101d5d4996939ead95a50312f4d8b3", + v1_cache="01GKS1X2Y18ECYRRJPCSGEE57X", + v2_cache="01GKS1X8BPMXMD9FQ4T1ER22YW", + ) + + known_surplus_fee = 1323758338760117 + # One incoming WETH and outgoing COW + self.assertEqual(2, len(v1_result)) + self.assertEqual(2, len(v2_result)) + weth_in = 82259811452690960 + cow_out = -1342108379340087200000 + + parsed_v1 = {v["symbol"]: v["amount"] for v in v1_result} + parsed_v2 = {v["symbol"]: float(v["amount"]) for v in v2_result} + + expected_tokens = {"WETH", "COW"} + self.assertEqual(expected_tokens, set(parsed_v1.keys())) + self.assertEqual(expected_tokens, set(parsed_v2.keys())) + # Something very weird about these types. + # Expected :-1342108379340087200000 + # Actual :-1.3421083793400872e+21 + self.assertAlmostEqual(cow_out, parsed_v1["COW"], places=18) + self.assertAlmostEqual(cow_out, parsed_v2["COW"], places=18) + + self.assertAlmostEqual( + # 13 WEI difference + weth_in - known_surplus_fee, + parsed_v1["WETH"], + delta=13, + ) + # V2 Query does not yet implement surplus fee. + self.assertAlmostEqual(weth_in, parsed_v2["WETH"], places=18) + if __name__ == "__main__": unittest.main()