Revert "fail scale-up batch when a single instance of the batch fails" (

#2448)
pytorch · Feb 7, 2023 · 9e8ad98 · 9e8ad98
1 parent 1b8c6c3
commit 9e8ad98
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 38 deletions.
diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/lambda.ts
@@ -1,7 +1,7 @@
 import { Context, SQSEvent, ScheduledEvent } from 'aws-lambda';
 
 import { scaleDown as scaleDownR } from './scale-runners/scale-down';
-import { scaleUp as scaleUpR, RetryableScalingError } from './scale-runners/scale-up';
+import { scaleUp as scaleUpR } from './scale-runners/scale-up';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export async function scaleUp(event: SQSEvent, context: Context, callback: any) {
@@ -13,12 +13,7 @@ export async function scaleUp(event: SQSEvent, context: Context, callback: any)
     return callback(null);
   } catch (e) {
     console.error(e);
-    if (e instanceof RetryableScalingError) {
-      console.error('Received a RetryableScalingError, will callback with failure');
-      return callback(e);
-    } else {
-      return callback('Failed handling SQS event');
-    }
+    return callback('Failed handling SQS event');
   }
 }
 

diff --git a/...form-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts b/...form-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts
@@ -17,6 +17,7 @@ import * as MetricsModule from './metrics';
 jest.mock('./runners');
 jest.mock('./gh-runners');
 jest.mock('./gh-issues');
+jest.mock('./metrics');
 
 beforeEach(() => {
   jest.resetModules();
@@ -29,7 +30,6 @@ const baseCfg = {
   awsRegion: 'us-east-1',
   cantHaveIssuesLabels: [],
   mustHaveIssuesLabels: [],
-  lambdaTimeout: 600,
 } as unknown as Config;
 
 const metrics = new MetricsModule.ScaleUpMetrics();

diff --git a/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts b/terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts
@@ -1,4 +1,4 @@
-import { Metrics, ScaleUpMetrics, sendMetricsAtTimeout, sendMetricsTimeoutVars } from './metrics';
+import { Metrics, ScaleUpMetrics } from './metrics';
 import { Repo, getRepoKey } from './utils';
 import { RunnerType, RunnerInputParameters, createRunner } from './runners';
 import {
@@ -21,32 +21,16 @@ export interface ActionRequestMessage {
   runnerLabels?: string[];
 }
 
-export class RetryableScalingError extends Error {
-  constructor(message: string) {
-    super(message);
-    this.name = 'RetryableScalingError';
-  }
-}
-
 export async function scaleUp(eventSource: string, payload: ActionRequestMessage): Promise<void> {
   if (eventSource !== 'aws:sqs') throw Error('Cannot handle non-SQS events!');
 
   const metrics = new ScaleUpMetrics();
-  const sndMetricsTimout: sendMetricsTimeoutVars = {
-    metrics: metrics,
-  };
-  sndMetricsTimout.setTimeout = setTimeout(
-    sendMetricsAtTimeout(sndMetricsTimout),
-    (Config.Instance.lambdaTimeout - 10) * 1000,
-  );
 
   const repo: Repo = {
     owner: payload.repositoryOwner,
     repo: payload.repositoryName,
   };
 
-  const errors = [];
-
   try {
     if (await shouldSkipForRepo(repo, metrics)) {
       metrics.skipRepo(repo);
@@ -102,8 +86,6 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage
             metrics.runnersRepoCreate(repo, runnerType.runnerTypeName, awsRegion);
           }
         } catch (e) {
-          errors.push(e);
-
           /* istanbul ignore next */
           if (Config.Instance.enableOrganizationRunners) {
             metrics.runnersOrgCreateFail(repo.owner, runnerType.runnerTypeName);
@@ -118,19 +100,8 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage
       }
     }
   } finally {
-    clearTimeout(sndMetricsTimout.setTimeout);
-    sndMetricsTimout.metrics = undefined;
-    sndMetricsTimout.setTimeout = undefined;
     metrics.sendMetrics();
   }
-
-  if (errors.length > 0) {
-    const msg =
-      `Thrown ${errors.length} exceptions during scaleup when creating runners, ` +
-      'will fail this batch so it can be retried';
-    console.warn(msg);
-    throw new RetryableScalingError(msg);
-  }
 }
 
 async function createRunnerConfigArgument(