feat(Dataworker): Support pre-fill refunds

This PR should be backwards compatible and deployable to production today

to be paired with across-protocol/sdk#835

test/Dataworker.loadData.fill.ts

@@ -590,9 +590,18 @@ describe("Dataworker: Load data used in all functions", async function () {
       expect(data1.bundleDepositsV3).to.deep.equal({});
     });
     it("Filters fills out of block range", async function () {
-      generateV3Deposit({ outputToken: randomAddress() });
-      generateV3Deposit({ outputToken: randomAddress() });
-      generateV3Deposit({ outputToken: randomAddress() });
+      generateV3Deposit({
+        outputToken: randomAddress(),
+        blockNumber: mockOriginSpokePoolClient.eventManager.blockNumber + 1,
+      });
+      generateV3Deposit({
+        outputToken: randomAddress(),
+        blockNumber: mockOriginSpokePoolClient.eventManager.blockNumber + 11,
+      });
+      generateV3Deposit({
+        outputToken: randomAddress(),
+        blockNumber: mockOriginSpokePoolClient.eventManager.blockNumber + 21,
+      });
       await mockOriginSpokePoolClient.update(["V3FundsDeposited"]);
       const deposits = mockOriginSpokePoolClient.getDeposits();
 
@@ -607,13 +616,16 @@ describe("Dataworker: Load data used in all functions", async function () {
           blockNumber: mockDestinationSpokePoolClient.eventManager.blockNumber + 21,
         }),
       ];
-      // Create a block range that contains only the middle event.
+      // Create a block range that contains only the middle events.
       const destinationChainBlockRange = [fills[1].blockNumber - 1, fills[1].blockNumber + 1];
-      // Substitute destination chain bundle block range.
+      const originChainBlockRange = [deposits[1].blockNumber - 1, deposits[1].blockNumber + 1];
+      // Substitute bundle block ranges.
       const bundleBlockRanges = getDefaultBlockRange(5);
       const destinationChainIndex =
         dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers.indexOf(destinationChainId);
       bundleBlockRanges[destinationChainIndex] = destinationChainBlockRange;
+      const originChainIndex = dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers.indexOf(originChainId);
+      bundleBlockRanges[originChainIndex] = originChainBlockRange;
       await mockDestinationSpokePoolClient.update(["FilledV3Relay"]);
       expect(mockDestinationSpokePoolClient.getFills().length).to.equal(fills.length);
       const data1 = await dataworkerInstance.clients.bundleDataClient.loadData(bundleBlockRanges, spokePoolClients);
@@ -725,8 +737,11 @@ describe("Dataworker: Load data used in all functions", async function () {
 
       const data1 = await dataworkerInstance.clients.bundleDataClient.loadData(blockRanges, spokePoolClients);
       expect(data1.bundleDepositsV3).to.deep.equal({});
-      expect(data1.bundleFillsV3[repaymentChainId][l1Token_1.address].fills.length).to.equal(1);
-      expect(spyLogIncludes(spy, -2, "invalid V3 fills in range")).to.be.false;
+
+      // Fill should not be included since we cannot validate fills when the deposit is in a following bundle.
+      // This fill is considered a "pre-fill" and will be validated when the deposit is included in a bundle.
+      expect(data1.bundleFillsV3).to.deep.equal({});
+      expect(spyLogIncludes(spy, -2, "invalid V3 fills in range")).to.be.true;
     });
     it("Does not count prior bundle expired deposits that were filled", async function () {
       // Send deposit that expires in this bundle.

test/Dataworker.loadData.prefill.ts

@@ -0,0 +1,331 @@
+import { BundleDataClient, ConfigStoreClient, HubPoolClient, SpokePoolClient } from "../src/clients";
+import { destinationChainId, originChainId, repaymentChainId } from "./constants";
+import { setupDataworker } from "./fixtures/Dataworker.Fixture";
+import {
+  Contract,
+  FakeContract,
+  SignerWithAddress,
+  V3FillFromDeposit,
+  ethers,
+  expect,
+  getDefaultBlockRange,
+  randomAddress,
+  smock,
+} from "./utils";
+
+import { Dataworker } from "../src/dataworker/Dataworker"; // Tested
+import { getCurrentTime, Event, toBNWei, ZERO_ADDRESS } from "../src/utils";
+import { MockHubPoolClient, MockSpokePoolClient } from "./mocks";
+import { interfaces } from "@across-protocol/sdk";
+
+let erc20_1: Contract, erc20_2: Contract;
+let l1Token_1: Contract;
+let relayer: SignerWithAddress;
+
+let spokePoolClient_1: SpokePoolClient, spokePoolClient_2: SpokePoolClient;
+let hubPoolClient: HubPoolClient, configStoreClient: ConfigStoreClient;
+let dataworkerInstance: Dataworker;
+let spokePoolClients: { [chainId: number]: SpokePoolClient };
+
+let updateAllClients: () => Promise<void>;
+
+describe("BundleDataClient: Pre-fill logic", async function () {
+  beforeEach(async function () {
+    ({
+      erc20_1,
+      erc20_2,
+      hubPoolClient,
+      configStoreClient,
+      l1Token_1,
+      relayer,
+      dataworkerInstance,
+      spokePoolClient_1,
+      spokePoolClient_2,
+      spokePoolClients,
+      updateAllClients,
+    } = await setupDataworker(ethers, 25, 25, 0));
+  });
+
+  describe("loadDataFromScratch", function () {
+    let mockOriginSpokePoolClient: MockSpokePoolClient, mockDestinationSpokePoolClient: MockSpokePoolClient;
+    let mockHubPoolClient: MockHubPoolClient;
+    let mockDestinationSpokePool: FakeContract;
+    const lpFeePct = toBNWei("0.01");
+
+    beforeEach(async function () {
+      await updateAllClients();
+      mockHubPoolClient = new MockHubPoolClient(
+        hubPoolClient.logger,
+        hubPoolClient.hubPool,
+        configStoreClient,
+        hubPoolClient.deploymentBlock,
+        hubPoolClient.chainId
+      );
+      // Mock a realized lp fee pct for each deposit so we can check refund amounts and bundle lp fees.
+      mockHubPoolClient.setDefaultRealizedLpFeePct(lpFeePct);
+      mockOriginSpokePoolClient = new MockSpokePoolClient(
+        spokePoolClient_1.logger,
+        spokePoolClient_1.spokePool,
+        spokePoolClient_1.chainId,
+        spokePoolClient_1.deploymentBlock
+      );
+      mockDestinationSpokePool = await smock.fake(spokePoolClient_2.spokePool.interface);
+      mockDestinationSpokePoolClient = new MockSpokePoolClient(
+        spokePoolClient_2.logger,
+        mockDestinationSpokePool as Contract,
+        spokePoolClient_2.chainId,
+        spokePoolClient_2.deploymentBlock
+      );
+      spokePoolClients = {
+        ...spokePoolClients,
+        [originChainId]: mockOriginSpokePoolClient,
+        [destinationChainId]: mockDestinationSpokePoolClient,
+      };
+      await mockHubPoolClient.update();
+      await mockOriginSpokePoolClient.update();
+      await mockDestinationSpokePoolClient.update();
+      mockHubPoolClient.setTokenMapping(l1Token_1.address, originChainId, erc20_1.address);
+      mockHubPoolClient.setTokenMapping(l1Token_1.address, destinationChainId, erc20_2.address);
+      mockHubPoolClient.setTokenMapping(l1Token_1.address, repaymentChainId, l1Token_1.address);
+      const bundleDataClient = new BundleDataClient(
+        dataworkerInstance.logger,
+        {
+          ...dataworkerInstance.clients.bundleDataClient.clients,
+          hubPoolClient: mockHubPoolClient as unknown as HubPoolClient,
+        },
+        dataworkerInstance.clients.bundleDataClient.spokePoolClients,
+        dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers
+      );
+      dataworkerInstance = new Dataworker(
+        dataworkerInstance.logger,
+        { ...dataworkerInstance.clients, bundleDataClient },
+        dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers,
+        dataworkerInstance.maxRefundCountOverride,
+        dataworkerInstance.maxL1TokenCountOverride,
+        dataworkerInstance.blockRangeEndBlockBuffer
+      );
+    });
+
+    function generateV3Deposit(eventOverride?: Partial<interfaces.DepositWithBlock>): Event {
+      return mockOriginSpokePoolClient.depositV3({
+        inputToken: erc20_1.address,
+        outputToken: eventOverride?.outputToken ?? erc20_2.address,
+        message: "0x",
+        quoteTimestamp: eventOverride?.quoteTimestamp ?? getCurrentTime() - 10,
+        fillDeadline: eventOverride?.fillDeadline ?? getCurrentTime() + 14400,
+        destinationChainId,
+        blockNumber: eventOverride?.blockNumber ?? spokePoolClient_1.latestBlockSearched, // @dev use latest block searched from non-mocked client
+        // so that mocked client's latestBlockSearched gets set to the same value.
+      } as interfaces.DepositWithBlock);
+    }
+
+    function generateV3FillFromDeposit(
+      deposit: interfaces.DepositWithBlock,
+      fillEventOverride?: Partial<interfaces.FillWithBlock>,
+      _relayer = relayer.address,
+      _repaymentChainId = repaymentChainId,
+      fillType = interfaces.FillType.FastFill
+    ): Event {
+      const fillObject = V3FillFromDeposit(deposit, _relayer, _repaymentChainId);
+      return mockDestinationSpokePoolClient.fillV3Relay({
+        ...fillObject,
+        relayExecutionInfo: {
+          updatedRecipient: fillObject.updatedRecipient,
+          updatedMessage: fillObject.updatedMessage,
+          updatedOutputAmount: fillObject.updatedOutputAmount,
+          fillType,
+        },
+        blockNumber: fillEventOverride?.blockNumber ?? spokePoolClient_2.latestBlockSearched, // @dev use latest block searched from non-mocked client
+        // so that mocked client's latestBlockSearched gets set to the same value.
+      } as interfaces.FillWithBlock);
+    }
+
+    function generateSlowFillRequestFromDeposit(
+      deposit: interfaces.DepositWithBlock,
+      fillEventOverride?: Partial<interfaces.FillWithBlock>
+    ): Event {
+      const fillObject = V3FillFromDeposit(deposit, ZERO_ADDRESS);
+      const { relayer, repaymentChainId, relayExecutionInfo, ...relayData } = fillObject;
+      return mockDestinationSpokePoolClient.requestV3SlowFill({
+        ...relayData,
+        blockNumber: fillEventOverride?.blockNumber ?? spokePoolClient_2.latestBlockSearched, // @dev use latest block searched from non-mocked client
+        // so that mocked client's latestBlockSearched gets set to the same value.
+      } as interfaces.SlowFillRequest);
+    }
+
+    describe("Pre-fills", function () {
+      it("Refunds fill if fill is in-memory and in older bundle", async function () {
+        generateV3Deposit({ outputToken: randomAddress() });
+        await mockOriginSpokePoolClient.update(["V3FundsDeposited"]);
+        const deposits = mockOriginSpokePoolClient.getDeposits();
+
+        // Submit fill that we won't include in the bundle block range.
+        const fill = generateV3FillFromDeposit(deposits[0], {
+          blockNumber: mockDestinationSpokePoolClient.eventManager.blockNumber,
+        });
+        // Substitute bundle block ranges.
+        const bundleBlockRanges = getDefaultBlockRange(5);
+        const destinationChainIndex =
+          dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers.indexOf(destinationChainId);
+        bundleBlockRanges[destinationChainIndex] = [fill.blockNumber + 1, fill.blockNumber + 2];
+
+        await mockDestinationSpokePoolClient.update(["FilledV3Relay"]);
+        expect(mockDestinationSpokePoolClient.getFills().length).to.equal(1);
+
+        // So, one of the fills is a pre-fill because its earlier than the bundle block range. Because its corresponding
+        // deposit is in the block range, we should refund it.
+        const data1 = await dataworkerInstance.clients.bundleDataClient.loadData(bundleBlockRanges, spokePoolClients);
+        expect(data1.bundleFillsV3[repaymentChainId][l1Token_1.address].fills.length).to.equal(1);
+        expect(data1.bundleFillsV3[repaymentChainId][l1Token_1.address].fills[0].depositId).to.equal(
+          fill.args.depositId
+        );
+      });
+
+      it("Does not refund fill if fill is in-memory but in a future bundle", async function () {
+        generateV3Deposit({ outputToken: randomAddress() });
+        await mockOriginSpokePoolClient.update(["V3FundsDeposited"]);
+        const deposits = mockOriginSpokePoolClient.getDeposits();
+
+        // Submit fill that we won't include in the bundle block range but is in a future bundle
+        const futureFill = generateV3FillFromDeposit(deposits[0], {
+          blockNumber: mockDestinationSpokePoolClient.eventManager.blockNumber + 11,
+        });
+
+        // Substitute bundle block ranges.
+        const bundleBlockRanges = getDefaultBlockRange(5);
+        const destinationChainIndex =
+          dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers.indexOf(destinationChainId);
+        bundleBlockRanges[destinationChainIndex] = [futureFill.blockNumber - 2, futureFill.blockNumber - 1];
+
+        await mockDestinationSpokePoolClient.update(["FilledV3Relay"]);
+        expect(mockDestinationSpokePoolClient.getFills().length).to.equal(1);
+
+        const data1 = await dataworkerInstance.clients.bundleDataClient.loadData(bundleBlockRanges, spokePoolClients);
+        expect(data1.bundleFillsV3).to.deep.equal({});
+      });
+
+      it("Does not refund fill if fill is in-memory but its a SlowFill", async function () {
+        generateV3Deposit({ outputToken: randomAddress() });
+        await mockOriginSpokePoolClient.update(["V3FundsDeposited"]);
+        const deposits = mockOriginSpokePoolClient.getDeposits();
+
+        // Submit fill in an older bundle but its a Slow Fill execution.
+        const slowFill = generateV3FillFromDeposit(
+          deposits[0],
+          { blockNumber: mockDestinationSpokePoolClient.eventManager.blockNumber },
+          undefined,
+          undefined,
+          interfaces.FillType.SlowFill
+        );
+
+        // Substitute bundle block ranges.
+        const bundleBlockRanges = getDefaultBlockRange(5);
+        const destinationChainIndex =
+          dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers.indexOf(destinationChainId);
+        bundleBlockRanges[destinationChainIndex] = [slowFill.blockNumber + 1, slowFill.blockNumber + 2];
+
+        await mockDestinationSpokePoolClient.update(["FilledV3Relay"]);
+        expect(mockDestinationSpokePoolClient.getFills().length).to.equal(1);
+
+        const data1 = await dataworkerInstance.clients.bundleDataClient.loadData(bundleBlockRanges, spokePoolClients);
+        expect(data1.bundleFillsV3).to.deep.equal({});
+      });
+
+      it("Refunds fill if fill status is Filled", async function () {
+        // Checks relayStatuses() for fill status
+        // Loads old event and sets refund address and repayment chain correctly.
+      });
+
+      it("Does not refund if fill status is not Filled", async function () {});
+    });
+
+    describe.only("Pre-slow-fill-requests", function () {
+      it("Creates slow fill leaf if slow fill request is in-memory and in older bundle", async function () {
+        generateV3Deposit({ outputToken: erc20_2.address });
+        await mockOriginSpokePoolClient.update(["V3FundsDeposited"]);
+        const deposits = mockOriginSpokePoolClient.getDeposits();
+
+        // Submit request that we won't include in the bundle block range.
+        const request = generateSlowFillRequestFromDeposit(deposits[0], {
+          blockNumber: mockDestinationSpokePoolClient.eventManager.blockNumber,
+        });
+        // Substitute bundle block ranges.
+        const bundleBlockRanges = getDefaultBlockRange(5);
+        const destinationChainIndex =
+          dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers.indexOf(destinationChainId);
+        bundleBlockRanges[destinationChainIndex] = [request.blockNumber + 1, request.blockNumber + 2];
+
+        await mockDestinationSpokePoolClient.update(["RequestedV3SlowFill"]);
+        expect(mockDestinationSpokePoolClient.getSlowFillRequestsForOriginChain(originChainId).length).to.equal(1);
+
+        // So, one of the fills is a pre-fill because its earlier than the bundle block range. Because its corresponding
+        // deposit is in the block range, we should refund it.
+        const data1 = await dataworkerInstance.clients.bundleDataClient.loadData(bundleBlockRanges, spokePoolClients);
+        expect(data1.bundleSlowFillsV3[destinationChainId][erc20_2.address].length).to.equal(1);
+        expect(data1.bundleSlowFillsV3[destinationChainId][erc20_2.address][0].depositId).to.equal(
+          request.args.depositId
+        );
+      });
+
+      it("Creates slow fill leaf if fill status is RequestedSlowFill", async function () {
+        // Checks relayStatuses() for fill status
+        // Creates slow fill leaf.
+      });
+
+      it("Does not create slow fill leaf if slow fill request is in-memory but in a future bundle", async function () {
+        generateV3Deposit({ outputToken: erc20_2.address });
+        await mockOriginSpokePoolClient.update(["V3FundsDeposited"]);
+        const deposits = mockOriginSpokePoolClient.getDeposits();
+
+        // Submit request that we won't include in the bundle block range but is in a future bundle
+        const request = generateSlowFillRequestFromDeposit(deposits[0], {
+          blockNumber: mockDestinationSpokePoolClient.eventManager.blockNumber + 11,
+        });
+
+        // Substitute bundle block ranges.
+        const bundleBlockRanges = getDefaultBlockRange(5);
+        const destinationChainIndex =
+          dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers.indexOf(destinationChainId);
+        bundleBlockRanges[destinationChainIndex] = [request.blockNumber - 2, request.blockNumber - 1];
+
+        await mockDestinationSpokePoolClient.update(["RequestedV3SlowFill"]);
+        expect(mockDestinationSpokePoolClient.getSlowFillRequestsForOriginChain(originChainId).length).to.equal(1);
+
+        const data1 = await dataworkerInstance.clients.bundleDataClient.loadData(bundleBlockRanges, spokePoolClients);
+        expect(data1.bundleSlowFillsV3).to.deep.equal({});
+      });
+
+      it("Does not create slow fill leaf if slow fill request is in-memory but an invalid request", async function () {
+        generateV3Deposit({ outputToken: randomAddress() });
+        generateV3Deposit({ outputToken: erc20_2.address, fillDeadline: 0 });
+        await mockOriginSpokePoolClient.update(["V3FundsDeposited"]);
+        const deposits = mockOriginSpokePoolClient.getDeposits();
+
+        // Submit request that that is in a previous bundle but is invalid.
+        generateSlowFillRequestFromDeposit(deposits[0], {
+          blockNumber: mockDestinationSpokePoolClient.eventManager.blockNumber,
+        });
+        const expiredDepositRequest = generateSlowFillRequestFromDeposit(deposits[1], {
+          blockNumber: mockDestinationSpokePoolClient.eventManager.blockNumber,
+        });
+
+        // Substitute bundle block ranges.
+        const bundleBlockRanges = getDefaultBlockRange(5);
+        const destinationChainIndex =
+          dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers.indexOf(destinationChainId);
+        bundleBlockRanges[destinationChainIndex] = [
+          expiredDepositRequest.blockNumber + 1,
+          expiredDepositRequest.blockNumber + 2,
+        ];
+
+        await mockDestinationSpokePoolClient.update(["RequestedV3SlowFill"]);
+        expect(mockDestinationSpokePoolClient.getSlowFillRequestsForOriginChain(originChainId).length).to.equal(2);
+
+        const data1 = await dataworkerInstance.clients.bundleDataClient.loadData(bundleBlockRanges, spokePoolClients);
+        expect(data1.bundleSlowFillsV3).to.deep.equal({});
+      });
+
+      it("Does not create slow fill leaf if fill status is not RequestedSlowFill", async function () {});
+    });
+  });
+});

test/Dataworker.loadData.slowFill.ts

@@ -463,9 +463,18 @@ describe("BundleDataClient: Slow fill handling & validation", async function ()
   });
 
   it("Handles slow fill requests out of block range", async function () {
-    generateV3Deposit({ outputToken: erc20_2.address });
-    generateV3Deposit({ outputToken: erc20_2.address });
-    generateV3Deposit({ outputToken: erc20_2.address });
+    generateV3Deposit({
+      outputToken: erc20_2.address,
+      blockNumber: mockOriginSpokePoolClient.eventManager.blockNumber + 1,
+    });
+    generateV3Deposit({
+      outputToken: erc20_2.address,
+      blockNumber: mockOriginSpokePoolClient.eventManager.blockNumber + 11,
+    });
+    generateV3Deposit({
+      outputToken: erc20_2.address,
+      blockNumber: mockOriginSpokePoolClient.eventManager.blockNumber + 21,
+    });
     await mockOriginSpokePoolClient.update(["V3FundsDeposited"]);
     const deposits = mockOriginSpokePoolClient.getDeposits();
 
@@ -480,13 +489,16 @@ describe("BundleDataClient: Slow fill handling & validation", async function ()
         blockNumber: mockDestinationSpokePoolClient.eventManager.blockNumber + 21,
       }),
     ];
-    // Create a block range that contains only the middle event.
+    // Create a block range that contains only the middle events.
     const destinationChainBlockRange = [events[1].blockNumber - 1, events[1].blockNumber + 1];
-    // Substitute destination chain bundle block range.
+    const originChainBlockRange = [deposits[1].blockNumber - 1, deposits[1].blockNumber + 1];
+    // Substitute bundle block ranges.
     const bundleBlockRanges = getDefaultBlockRange(5);
     const destinationChainIndex =
       dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers.indexOf(destinationChainId);
     bundleBlockRanges[destinationChainIndex] = destinationChainBlockRange;
+    const originChainIndex = dataworkerInstance.chainIdListForBundleEvaluationBlockNumbers.indexOf(originChainId);
+    bundleBlockRanges[originChainIndex] = originChainBlockRange;
     await mockDestinationSpokePoolClient.update(["RequestedV3SlowFill"]);
     expect(mockDestinationSpokePoolClient.getSlowFillRequestsForOriginChain(originChainId).length).to.equal(3);
     const data1 = await dataworkerInstance.clients.bundleDataClient.loadData(bundleBlockRanges, spokePoolClients);