[V1] Add surplus_fee to slippage query (#156)

Separating #155 into two "independent" PRs so we can add tests n stuff.

The slippage query has been altered to account for surplus_fee. A new test is added, demonstrating that this surplus fee is captured by the V1 query and not the V2 query (since this is not yet possible).

Note that while testing this, I have discovered a bug in our base slippage query. Receiver = 0 There are only 113 total trades over all time with this set (see here) which had to be fixed before we could properly test this. That is because the query thought that there was an AMM_OUT transfer.

queries/dune_v1/period_slippage.sql

@@ -8,14 +8,21 @@ filtered_trades as (
            solver_name,
            solver_address,
            trader                                                as trader_in,
-           receiver                                              as trader_out,
+           -- Null Receiver: https://dune.com/queries/1729130
+           case
+                when receiver = '\x0000000000000000000000000000000000000000'
+                then trader
+            else receiver end                                    as trader_out,
            sell_token_address                                    as "sellToken",
            buy_token_address                                     as "buyToken",
-           atoms_sold                                            as "sellAmount",
+           atoms_sold - coalesce(surplus_fee, 0)                 as "sellAmount",
            atoms_bought                                          as "buyAmount",
            '\x9008D19f58AAbD9eD0D60971565AA8510560ab41' :: bytea as contract_address
     from gnosis_protocol_v2."trades" t
              join gnosis_protocol_v2."batches" b on t.tx_hash = b.tx_hash
+    left outer join dune_user_generated.cow_order_rewards_barn f
+        on f.tx_hash = t.tx_hash
+        and f.order_uid = t.order_uid
     where b.block_time between '{{StartTime}}'
         and '{{EndTime}}'
       and case

queries/dune_v2/period_slippage.sql

@@ -11,7 +11,12 @@ filtered_trades as (
            num_trades,
            solver_address,
            trader                                              as trader_in,
-           receiver                                            as trader_out,
+           -- Null Receiver: https://dune.com/queries/1729130
+           -- TODO - This could also be fixed in the spellbook!
+           case
+                when receiver = '0x0000000000000000000000000000000000000000'
+                then trader
+            else receiver end                                  as trader_out,
            sell_token_address                                  as sell_token,
            buy_token_address                                   as buy_token,
            atoms_sold,

src/queries.py

@@ -95,7 +95,7 @@ def with_params(
     "PERIOD_SLIPPAGE": QueryData(
         name="Solver Slippage for Period",
         filepath="period_slippage.sql",
-        v1_id=1570227,
-        v2_id=1570561,
+        v1_id=1728478,
+        v2_id=1729274,
     ),
 }

src/update/reward_history.py

@@ -57,9 +57,7 @@ def __str__(self) -> str:
         solver, tx_hash, order_id = list(
             map(pg_hex2bytea, [self.solver, self.tx_hash, self.order_uid])
         )
-        return (
-            f"('{order_id}','{solver}','{tx_hash}', {self.surplus_fee},{self.amount})"
-        )
+        return f"('{order_id}','{solver}','{tx_hash}',{self.surplus_fee},{self.amount})"
 
 
 def fetch_and_push_order_rewards(dune: DuneAPI, env: Environment) -> None:

tests/integration/common.py

@@ -2,6 +2,10 @@
 
 from dune_client.client import DuneClient
 from dune_client.query import Query
+from dune_client.types import QueryParameter
+
+from src.models.accounting_period import AccountingPeriod
+from src.queries import QUERIES
 
 
 def exec_or_get(dune: DuneClient, query: Query, result_id: Optional[str] = None):
@@ -10,3 +14,36 @@ def exec_or_get(dune: DuneClient, query: Query, result_id: Optional[str] = None)
         print(f"Execution ID: {results.execution_id}")
         return results
     return dune.get_result(result_id)
+
+
+def get_slippage_cte_rows(
+    dune: DuneClient,
+    cte_name: str,
+    period: AccountingPeriod,
+    tx_hash: Optional[str] = None,
+    v1_cache: Optional[str] = None,
+    v2_cache: Optional[str] = None,
+):
+    slippage_query = QUERIES["PERIOD_SLIPPAGE"]
+    v1_query_id = slippage_query.v1_query.query_id
+    v2_query_id = slippage_query.v2_query.query_id
+
+    parameters = period.as_query_params()
+    parameters.append(QueryParameter.enum_type("CTE_NAME", cte_name))
+    if tx_hash:
+        parameters.append(QueryParameter.text_type("TxHash", tx_hash))
+
+    v1_results = exec_or_get(
+        dune,
+        Query(v1_query_id, params=parameters),
+        v1_cache,
+    )
+    v2_results = exec_or_get(
+        dune,
+        Query(v2_query_id, params=parameters),
+        v2_cache,
+    )
+
+    v1_rows = v1_results.get_rows()
+    v2_rows = v2_results.get_rows()
+    return v1_rows, v2_rows

tests/integration/test_solver_slippage.py

@@ -7,14 +7,12 @@
 
 from dune_client.client import DuneClient
 from dune_client.file.interface import FileIO
-from dune_client.query import Query
-from dune_client.types import QueryParameter
 
 from src.constants import FILE_OUT_DIR
 from src.models.accounting_period import AccountingPeriod
 from src.models.slippage import SplitSlippages
 from src.queries import QUERIES
-from tests.integration.common import exec_or_get
+from tests.integration.common import get_slippage_cte_rows
 
 
 @dataclass
@@ -75,49 +73,26 @@ def setUp(self) -> None:
         self.parameters = self.period.as_query_params()
         self.writer = FileIO(FILE_OUT_DIR)
 
-    def get_cte_rows(
-        self,
-        cte_name: str,
-        tx_hash: Optional[str] = None,
-        v1_cache: Optional[str] = None,
-        v2_cache: Optional[str] = None,
-    ):
-        parameters = self.period.as_query_params()
-        parameters.append(QueryParameter.enum_type("CTE_NAME", cte_name))
-        if tx_hash:
-            parameters.append(QueryParameter.text_type("TxHash", tx_hash))
-
-        v1_results = exec_or_get(
-            self.dune,
-            Query(self.slippage_query.v1_query.query_id, params=parameters),
-            v1_cache,
-        )
-        v2_results = exec_or_get(
-            self.dune,
-            Query(self.slippage_query.v2_query.query_id, params=parameters),
-            v2_cache,
-        )
-
-        v1_rows = v1_results.get_rows()
-        v2_rows = v2_results.get_rows()
-        return v1_rows, v2_rows
-
     def get_cte_results(
         self,
         cte_name: str,
         tx_hash: Optional[str] = None,
         v1_cache: Optional[str] = None,
         v2_cache: Optional[str] = None,
     ):
-        v1_rows, v2_rows = self.get_cte_rows(cte_name, tx_hash, v1_cache, v2_cache)
+        v1_rows, v2_rows = get_slippage_cte_rows(
+            self.dune, cte_name, self.period, tx_hash, v1_cache, v2_cache
+        )
         self.writer.write_csv(v1_rows, f"{cte_name}_{tx_hash}_v1.csv")
         self.writer.write_csv(v2_rows, f"{cte_name}_{tx_hash}_v2.csv")
         return Comparison.from_dune_results(v1_rows, v2_rows)
 
     def test_batch_transfers(self):
         table_name = "batch_transfers"
-        data = self.get_cte_results(
+        data = get_slippage_cte_rows(
+            self.dune,
             table_name,
+            self.period,
             v1_cache="01GJQNSER0PYSMGHMEBDT03805",
             v2_cache="01GJQNSER0PYSMGHMEBDT03805",
         )
@@ -138,8 +113,10 @@ def test_batch_transfers(self):
     def test_incoming_and_outgoing(self):
         # This test demonstrates that records are "essentially" matching up to this table.
         table_name = "incoming_and_outgoing"
-        data = self.get_cte_results(
+        data = get_slippage_cte_rows(
+            self.dune,
             table_name,
+            self.period,
             v1_cache="01GJR1HKR37V7HRPTRJCDMZBAX",
             v2_cache="01GJR1HTNS87RKTV90WKH4TVSC",
         )
@@ -171,8 +148,10 @@ def test_incoming_and_outgoing(self):
 
     def test_final_token_balance_sheet(self):
         table_name = "final_token_balance_sheet"
-        data = self.get_cte_results(
+        data = get_slippage_cte_rows(
+            self.dune,
             table_name,
+            self.period,
             # Results for Period(2022-11-01)
             # v1_not_v2 = 172 batches
             # v2_not_v1 = 107 batches
@@ -214,8 +193,10 @@ def test_final_token_balance_sheet(self):
 
     def test_similar_slippage_for_period(self):
         table_name = "results"
-        v1_result, v2_result = self.get_cte_rows(
+        v1_result, v2_result = get_slippage_cte_rows(
+            self.dune,
             table_name,
+            self.period,
             # ---------------------------------------
             # Results for Period(2022-11-01)
             # Negative: 0.0094 difference --> 9.4 USD
@@ -257,6 +238,46 @@ def test_similar_slippage_for_period(self):
             delta=delta,  # |v1 - v2| ~ 0.004  --> 4 USD (with ETH at 1000)
         )
 
+    def test_limit_order_slippage(self):
+        """This is an internal buffer trade containing a limit order (with a surplus_fee)"""
+        table_name = "incoming_and_outgoing"
+        v1_result, v2_result = get_slippage_cte_rows(
+            self.dune,
+            table_name,
+            period=AccountingPeriod("2022-11-29", 1),
+            tx_hash="0xfe4589525c1ed764273fbca9120b0e5f7f101d5d4996939ead95a50312f4d8b3",
+            v1_cache="01GKS1X2Y18ECYRRJPCSGEE57X",
+            v2_cache="01GKS1X8BPMXMD9FQ4T1ER22YW",
+        )
+
+        known_surplus_fee = 1323758338760117
+        # One incoming WETH and outgoing COW
+        self.assertEqual(2, len(v1_result))
+        self.assertEqual(2, len(v2_result))
+        weth_in = 82259811452690960
+        cow_out = -1342108379340087200000
+
+        parsed_v1 = {v["symbol"]: v["amount"] for v in v1_result}
+        parsed_v2 = {v["symbol"]: float(v["amount"]) for v in v2_result}
+
+        expected_tokens = {"WETH", "COW"}
+        self.assertEqual(expected_tokens, set(parsed_v1.keys()))
+        self.assertEqual(expected_tokens, set(parsed_v2.keys()))
+        # Something very weird about these types.
+        # Expected :-1342108379340087200000
+        # Actual   :-1.3421083793400872e+21
+        self.assertAlmostEqual(cow_out, parsed_v1["COW"], places=18)
+        self.assertAlmostEqual(cow_out, parsed_v2["COW"], places=18)
+
+        self.assertAlmostEqual(
+            # 13 WEI difference
+            weth_in - known_surplus_fee,
+            parsed_v1["WETH"],
+            delta=13,
+        )
+        # V2 Query does not yet implement surplus fee.
+        self.assertAlmostEqual(weth_in, parsed_v2["WETH"], places=18)
+
 
 if __name__ == "__main__":
     unittest.main()