Create the issue only when there are consecutive failures (#7485)

.github/workflows/functional-test.yaml

@@ -80,6 +80,8 @@ env:
   FUNCTIONAL_TEST_APP_ID: 425843
   # Private Git repository where terraform module for testing is stored.
   TF_RECIPE_PRIVATE_GIT_SOURCE: "git::https://github.com/radius-project/terraform-private-modules//kubernetes-redis"
+  # The number of failed tests to report.
+  ISSUE_CREATE_THRESHOLD: 2
 
 jobs:
   build:
@@ -694,14 +696,38 @@ jobs:
     runs-on: ubuntu-latest
     if: failure() && github.event_name == 'schedule' && github.repository == 'radius-project/radius'
     steps:
+      - name: Count recently failed tests
+        id: count_failures
+        uses: actions/github-script@v7
+        with:
+          script: |
+            response = await github.rest.actions.listWorkflowRuns({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'functional-test.yaml',
+              event: 'schedule',
+              per_page: 10
+            });
+            
+            failureCount = 1;
+            for (const run of response.data.workflow_runs) {
+              if (run.conclusion === 'failure') {
+                failureCount++;
+              } else {
+                break;
+              }
+            }
+            return failureCount;
       - name: Create failure issue for failing scheduled run
         uses: actions/github-script@v7
+        # Only create an issue if there are (env.ISSUE_CREATE_THRESHOLD) failures of the recent tests.
+        if: steps.count_failures.outputs.result >= env.ISSUE_CREATE_THRESHOLD
         with:
           github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
           script: |
             github.rest.issues.create({
               ...context.repo,
               title: `Scheduled functional test failed - Run ID: ${context.runId}`,
               labels: ['bug', 'test-failure'],
-              body: `## Bug information \n\nThis bug is generated automatically if the scheduled functional test fails. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
+              body: `## Bug information \n\nThis bug is generated automatically if the scheduled functional test fails more than ${process.env.ISSUE_CREATE_THRESHOLD} times. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
             })

.github/workflows/long-running-azure.yaml

@@ -92,6 +92,9 @@ env:
   # The current GitHub action link
   ACTION_LINK: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
 
+  # The number of failed tests to report.
+  ISSUE_CREATE_THRESHOLD: 2
+
 jobs:
   build:
     name: Build Radius for test
@@ -509,14 +512,38 @@ jobs:
     runs-on: ubuntu-latest
     if: failure() && github.repository == 'radius-project/radius' && github.event_name == 'schedule'
     steps:
+      - name: Count recently failed tests
+        id: count_failures
+        uses: actions/github-script@v7
+        with:
+          script: |
+            response = await github.rest.actions.listWorkflowRuns({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'long-running-azure.yaml',
+              event: 'schedule',
+              per_page: 10
+            });
+            
+            failureCount = 1;
+            for (const run of response.data.workflow_runs) {
+              if (run.conclusion === 'failure') {
+                failureCount++;
+              } else {
+                break;
+              }
+            }
+            return failureCount;
       - name: Create failure issue for failing long running test run
         uses: actions/github-script@v7
+        # Only create an issue if there are (env.ISSUE_CREATE_THRESHOLD) failures of the recent tests.
+        if: steps.count_failures.outputs.result >= env.ISSUE_CREATE_THRESHOLD
         with:
           github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
           script: |
             github.rest.issues.create({
               ...context.repo,
               title: `Scheduled long running test failed - Run ID: ${context.runId}`,
               labels: ['bug', 'test-failure'],
-              body: `## Bug information \n\nThis bug is generated automatically if the scheduled long running test fails. The Radius long running test operates on a schedule of every 2 hours everyday. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
+              body: `## Bug information \n\nThis bug is generated automatically if the scheduled long running test fails more than ${process.env.ISSUE_CREATE_THRESHOLD} times. The Radius long running test operates on a schedule of every 2 hours everyday. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
             })