improve(relayer): Improve resilience against non-updated SpokePoolClient

The BundleDataClient asserts that all SpokePoolClient instances have
been updated. Ideally it could handle a non-updated SpokePoolClient, but
in advance of that, filter the relayer's set of SpokePoolClients to
exclude any that have not been updated after some defined time period.

src/relayer/index.ts

@@ -47,8 +47,8 @@ export async function runRelayer(_logger: winston.Logger, baseSigner: Signer): P
   const relayer = new Relayer(await baseSigner.getAddress(), logger, relayerClients, config);
   await relayer.init();
 
-  const { spokePoolClients } = relayerClients;
   const simulate = !sendingRelaysEnabled;
+  let { spokePoolClients } = relayerClients;
   let txnReceipts: { [chainId: number]: Promise<string[]> } = {};
 
   try {
@@ -61,14 +61,32 @@ export async function runRelayer(_logger: winston.Logger, baseSigner: Signer): P
       const ready = await relayer.update();
       const activeRelayer = redis ? await redis.get(botIdentifier) : undefined;
 
-      // If there is another active relayer, allow up to 10 update cycles for this instance to be ready,
+      // If there is another active relayer, allow up to 20 update cycles for this instance to be ready,
       // then proceed unconditionally to protect against any RPC outages blocking the relayer.
-      if (!ready && activeRelayer && run < 10) {
-        const runTime = Math.round((performance.now() - tLoopStart) / 1000);
-        const delta = pollingDelay - runTime;
-        logger.debug({ at: "Relayer#run", message: `Not ready to relay, waiting ${delta} seconds.` });
-        await delay(delta);
-        continue;
+      if (!ready && activeRelayer) {
+        if (run < 20) {
+          const runTime = Math.round((performance.now() - tLoopStart) / 1000);
+          const delta = pollingDelay - runTime;
+          logger.debug({ at: "Relayer#run", message: `Not ready to relay, waiting ${delta} seconds.` });
+          await delay(delta);
+          continue;
+        }
+
+        // Some SpokePoolClients have not updated. Filter out the ones that are not updated to avoid upsetting the
+        // BundleDataClient.
+        let droppedSpokePools: string[] = [];
+        spokePoolClients = relayerClients.spokePoolClients = Object.fromEntries(
+          Object.values(relayerClients.spokePoolClients)
+            .filter(({ chainId, isUpdated }) => {
+              if (!isUpdated) {
+                droppedSpokePools.push(getNetworkName(chainId));
+              }
+
+              return isUpdated;
+           })
+           .map((spokePoolClient) => [spokePoolClient.chainId, spokePoolClient])
+        );
+        logger.warn({ at: "Relayer#run", message: "Proceeding without some SpokePools", droppedSpokePools });
       }
 
       // Signal to any existing relayer that a handover is underway, or alternatively