[drci] Exclude lint from broken trunk (#5717)

Fixes #5596

The current log classifier/line capture mechanism allows real failures
to be hidden by existing failures, resulting in multiple lint breakages.
Lint is a fast and stable job so we should force people to rebase

torchci/lib/drciUtils.ts

@@ -49,6 +49,7 @@ export const EXCLUDED_FROM_FLAKINESS = [
   "/ build",
   "check labels",
 ];
+export const EXCLUDED_FROM_BROKEN_TRUNK = ["lint"];
 // If the base commit is too old, don't query for similar failures because
 // it increases the risk of getting misclassification. This guardrail can
 // be relaxed once we achieve better accuracy from the log classifier. This
@@ -408,19 +409,27 @@ export async function isLogClassifierFailed(
   return job.conclusion === "failure" && (!hasFailureLines || !hasLog);
 }
 
-export function isExcludedFromFlakiness(job: RecentWorkflowsData): boolean {
-  // Lintrunner job are generally stable and should be excluded from flakiness
-  // detection
+function isExcluded(job: RecentWorkflowsData, excludedJobs: string[]): boolean {
   return (
     _.find(
-      EXCLUDED_FROM_FLAKINESS,
+      excludedJobs,
       (exclude: string) =>
         job.name !== "" &&
         job.name.toLowerCase().includes(exclude.toLowerCase())
     ) !== undefined
   );
 }
 
+export function isExcludedFromBrokenTrunk(job: RecentWorkflowsData): boolean {
+  // Lintrunner job are generally stable and should be excluded from broken trunk
+  // detection
+  return isExcluded(job, EXCLUDED_FROM_BROKEN_TRUNK);
+}
+
+export function isExcludedFromFlakiness(job: RecentWorkflowsData): boolean {
+  return isExcluded(job, EXCLUDED_FROM_FLAKINESS);
+}
+
 export async function fetchIssueLabels(
   octokit: Octokit,
   owner: string,

torchci/pages/api/drci/drci.ts

@@ -16,6 +16,7 @@ import {
   hasSimilarFailures,
   hasSimilarFailuresInSamePR,
   HUD_URL,
+  isExcludedFromBrokenTrunk,
   isExcludedFromFlakiness,
   isExcludedFromSimilarityPostProcessing,
   isInfraFlakyJob,
@@ -859,6 +860,11 @@ export async function getWorkflowJobsStatuses(
         continue;
       }
 
+      if (isExcludedFromBrokenTrunk(job)) {
+        failedJobs.push(job);
+        continue;
+      }
+
       const trunkFailure = getTrunkFailure(job, baseJobs);
       if (trunkFailure !== undefined) {
         brokenTrunkJobs.push(job);

torchci/test/drci.test.ts

@@ -75,7 +75,7 @@ const pendingA = getDummyJob({
 });
 
 const failedA = getDummyJob({
-  name: "Lint",
+  name: "somethingA",
   conclusion: "failure",
   completed_at: "2022-07-13 19:34:03",
   html_url: "a",
@@ -88,7 +88,7 @@ const failedA = getDummyJob({
 });
 
 const failedASuccessfulRetry = getDummyJob({
-  name: "Lint",
+  name: "somethingA",
   conclusion: "success",
   completed_at: "2022-07-14 19:34:03",
   html_url: "a",
@@ -100,7 +100,7 @@ const failedASuccessfulRetry = getDummyJob({
 });
 
 const failedAFailedRetry = getDummyJob({
-  name: "Lint",
+  name: "somethingA",
   conclusion: "failure",
   completed_at: "2022-07-15 19:34:03",
   html_url: "a",
@@ -384,10 +384,11 @@ describe("Update Dr. CI Bot Unit Tests", () => {
 
     expect(failureInfo.includes("3 New Failures, 1 Pending")).toBeTruthy();
     expect(failureInfo.includes(failedJobName)).toBeTruthy();
-    const expectedFailureOrder = `* [Lint](${HUD_URL}/pr/pytorch/pytorch/123#1) ([gh](a))
-    \`mind blown\`
+    const expectedFailureOrder = `
 * [something](${HUD_URL}/pr/pytorch/pytorch/123#1) ([gh](a))
     \`cde\`
+* [somethingA](${HUD_URL}/pr/pytorch/pytorch/123#1) ([gh](a))
+    \`mind blown\`
 * [z-docs / build-docs (cpp)](${HUD_URL}/pr/pytorch/pytorch/123#1) ([gh](a))
     \`bababa\``;
     expect(failureInfo.includes(expectedFailureOrder)).toBeTruthy();

torchci/test/drciUtils.test.ts

@@ -465,6 +465,17 @@ describe("Test various utils used by Dr.CI", () => {
     expect(isExcludedFromFlakiness(notExcludedJob)).toEqual(false);
   });
 
+  test("test isExcludedFromBrokenTrunk", () => {
+    const notExcludedJob: RecentWorkflowsData = getDummyJob({});
+    expect(drciUtils.isExcludedFromBrokenTrunk(notExcludedJob)).toEqual(false);
+
+    const excludedJob: RecentWorkflowsData = {
+      ...notExcludedJob,
+      name: "LinT / quick-checks / linux-job",
+    };
+    expect(drciUtils.isExcludedFromBrokenTrunk(excludedJob)).toEqual(true);
+  });
+
   test("test getSuppressedLabels", () => {
     const job: RecentWorkflowsData = getDummyJob({
       jobName: "not suppressed job",