From 7717483b5118ce35b55ad92959e8a3392da58304 Mon Sep 17 00:00:00 2001
From: Paul <108695806+pxrl@users.noreply.github.com>
Date: Tue, 24 Sep 2024 19:14:13 +0200
Subject: [PATCH] improve(relayer): Improve resilience against non-updated
 SpokePoolClient

The BundleDataClient asserts that all SpokePoolClient instances have
been updated. Ideally it could handle a non-updated SpokePoolClient, but
in advance of that, filter the relayer's set of SpokePoolClients to
exclude any that have not been updated after some defined time period.
---
 src/relayer/index.ts | 34 ++++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/src/relayer/index.ts b/src/relayer/index.ts
index da8413ee6e..10bb0ed461 100644
--- a/src/relayer/index.ts
+++ b/src/relayer/index.ts
@@ -47,8 +47,8 @@ export async function runRelayer(_logger: winston.Logger, baseSigner: Signer): P
   const relayer = new Relayer(await baseSigner.getAddress(), logger, relayerClients, config);
   await relayer.init();
 
-  const { spokePoolClients } = relayerClients;
   const simulate = !sendingRelaysEnabled;
+  let { spokePoolClients } = relayerClients;
   let txnReceipts: { [chainId: number]: Promise<string[]> } = {};
 
   try {
@@ -61,14 +61,32 @@ export async function runRelayer(_logger: winston.Logger, baseSigner: Signer): P
       const ready = await relayer.update();
       const activeRelayer = redis ? await redis.get(botIdentifier) : undefined;
 
-      // If there is another active relayer, allow up to 10 update cycles for this instance to be ready,
+      // If there is another active relayer, allow up to 20 update cycles for this instance to be ready,
       // then proceed unconditionally to protect against any RPC outages blocking the relayer.
-      if (!ready && activeRelayer && run < 10) {
-        const runTime = Math.round((performance.now() - tLoopStart) / 1000);
-        const delta = pollingDelay - runTime;
-        logger.debug({ at: "Relayer#run", message: `Not ready to relay, waiting ${delta} seconds.` });
-        await delay(delta);
-        continue;
+      if (!ready && activeRelayer) {
+        if (run < 20) {
+          const runTime = Math.round((performance.now() - tLoopStart) / 1000);
+          const delta = pollingDelay - runTime;
+          logger.debug({ at: "Relayer#run", message: `Not ready to relay, waiting ${delta} seconds.` });
+          await delay(delta);
+          continue;
+        }
+
+        // Some SpokePoolClients have not updated. Filter out the ones that are not updated to avoid upsetting the
+        // BundleDataClient.
+        let droppedSpokePools: string[] = [];
+        spokePoolClients = relayerClients.spokePoolClients = Object.fromEntries(
+          Object.values(relayerClients.spokePoolClients)
+            .filter(({ chainId, isUpdated }) => {
+              if (!isUpdated) {
+                droppedSpokePools.push(getNetworkName(chainId));
+              }
+
+              return isUpdated;
+           })
+           .map((spokePoolClient) => [spokePoolClient.chainId, spokePoolClient])
+        );
+        logger.warn({ at: "Relayer#run", message: "Proceeding without some SpokePools", droppedSpokePools });
       }
 
       // Signal to any existing relayer that a handover is underway, or alternatively