From 07ad76bef47eafa0f6aa79100b7f8e2b4452e01f Mon Sep 17 00:00:00 2001
From: Ford <ford@thegraph.com>
Date: Fri, 26 Jan 2024 10:51:51 -0800
Subject: [PATCH] agent,common: Move subgraph health safety check to
 reconcileActions

- Previously the safety check occurred later in the process. We should
identify early that a subgraph is failed and safety = true to not
add the action to the queue in the first place. The other benefit
of moving this check is so it has access to the corresponding rule
so the safety property can be used here for control.
---
 packages/indexer-agent/src/agent.ts           | 88 +++++++++++++------
 .../src/indexer-management/allocations.ts     |  6 --
 2 files changed, 60 insertions(+), 34 deletions(-)

diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts
index 2d2033cf2..8b2c51a33 100644
--- a/packages/indexer-agent/src/agent.ts
+++ b/packages/indexer-agent/src/agent.ts
@@ -1073,36 +1073,68 @@ export class Agent {
           epoch,
         )
       case true: {
-        // If no active allocations, create one
+        // If no active allocations and subgraph health passes safety check, create one
+        const indexingStatuses = await this.graphNode.indexingStatus([
+          deploymentAllocationDecision.deployment,
+        ])
+        const indexingStatus = indexingStatuses.find(
+          status =>
+            status.subgraphDeployment ==
+            deploymentAllocationDecision.deployment,
+        )
+        const failsHealthCheck =
+          (indexingStatus &&
+            indexingStatus.health == 'failed' &&
+            deploymentAllocationDecision.ruleMatch.rule?.safety) ||
+          !indexingStatus
         if (activeDeploymentAllocations.length === 0) {
-          // Fetch the latest closed allocation, if any
-          const mostRecentlyClosedAllocation = (
-            await network.networkMonitor.closedAllocations(
-              deploymentAllocationDecision.deployment,
+          if (failsHealthCheck) {
+            logger.warn(
+              'Subgraph deployment has failed health check, skipping allocate',
+              {
+                indexingStatus,
+                safety: deploymentAllocationDecision.ruleMatch.rule?.safety,
+              },
             )
-          )[0]
-          return await operator.createAllocation(
-            logger,
-            deploymentAllocationDecision,
-            mostRecentlyClosedAllocation,
-          )
-        }
-
-        // Refresh any expiring allocations
-        const expiringAllocations = await this.identifyExpiringAllocations(
-          logger,
-          activeDeploymentAllocations,
-          deploymentAllocationDecision,
-          epoch,
-          maxAllocationEpochs,
-          network,
-        )
-        if (expiringAllocations.length > 0) {
-          await operator.refreshExpiredAllocations(
-            logger,
-            deploymentAllocationDecision,
-            expiringAllocations,
-          )
+          } else {
+            // Fetch the latest closed allocation, if any
+            const mostRecentlyClosedAllocation = (
+              await network.networkMonitor.closedAllocations(
+                deploymentAllocationDecision.deployment,
+              )
+            )[0]
+            return await operator.createAllocation(
+              logger,
+              deploymentAllocationDecision,
+              mostRecentlyClosedAllocation,
+            )
+          }
+        } else if (activeDeploymentAllocations.length > 0) {
+          if (failsHealthCheck) {
+            return await operator.closeEligibleAllocations(
+              logger,
+              deploymentAllocationDecision,
+              activeDeploymentAllocations,
+              epoch,
+            )
+          } else {
+            // Refresh any expiring allocations
+            const expiringAllocations = await this.identifyExpiringAllocations(
+              logger,
+              activeDeploymentAllocations,
+              deploymentAllocationDecision,
+              epoch,
+              maxAllocationEpochs,
+              network,
+            )
+            if (expiringAllocations.length > 0) {
+              await operator.refreshExpiredAllocations(
+                logger,
+                deploymentAllocationDecision,
+                expiringAllocations,
+              )
+            }
+          }
         }
       }
     }
diff --git a/packages/indexer-common/src/indexer-management/allocations.ts b/packages/indexer-common/src/indexer-management/allocations.ts
index 9f53d970d..1fd78e540 100644
--- a/packages/indexer-common/src/indexer-management/allocations.ts
+++ b/packages/indexer-common/src/indexer-management/allocations.ts
@@ -356,12 +356,6 @@ export class AllocationManager {
         `Subgraph deployment, '${deployment.ipfsHash}', is not syncing`,
       )
     }
-    if (status && status.health == 'failed') {
-      throw indexerError(
-        IndexerErrorCode.IE020,
-        `Subgraph deployment, '${deployment.ipfsHash}', failed during syncing`,
-      )
-    }
 
     logger.debug('Obtain a unique Allocation ID')
     const { allocationSigner, allocationId } = uniqueAllocationID(