From 396057de431265ea704663f3ac5e7569d839dd9d Mon Sep 17 00:00:00 2001
From: Ron <45816308+rjaegers@users.noreply.github.com>
Date: Thu, 7 Nov 2024 12:11:15 +0100
Subject: [PATCH] test: reduce acceptance test flakiness (#632)

* test: make plugin Locator match exact

* test: isolate Playwright assertions in the Page Object Model

* test: create Codespace as early as possible to give more time for startup

* test: start codespace after checkout

* test: wait for codespace to be active before starting test

* ci: fix indentation of acceptance-test.yml

* ci: use correct token to wait for codespace
---
 .github/workflows/acceptance-test.yml      | 36 ++++++++++++++++++----
 test/cpp/features/pages/codespace.pom.ts   |  6 +++-
 test/cpp/features/steps/codespace.steps.ts |  3 +-
 3 files changed, 36 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/acceptance-test.yml b/.github/workflows/acceptance-test.yml
index c3f40b3d..718a0205 100644
--- a/.github/workflows/acceptance-test.yml
+++ b/.github/workflows/acceptance-test.yml
@@ -25,20 +25,44 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
-      - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
-        with:
-          node-version: 20
-      - run: npm ci
-      - run: npx playwright install --with-deps
       # Create a GitHub Codespace and communicate the image version via a Codespace secret (should be a Codespace environment variable).
       # This secret is used by devcontainer.json, as such it is a resource that should not be used concurrently.
-      - run: |
+      - name: Start Codespace
+        run: |
           set -Eeuo pipefail
           gh secret set -a codespaces IMAGE_VERSION --body "pr-${{ github.event.pull_request.number }}"
           echo CODESPACE_NAME="$(gh codespace create -R "${{ github.repository }}" -b "$HEAD_REF" -m basicLinux32gb --devcontainer-path ".devcontainer/${{ inputs.flavor }}-test/devcontainer.json" --idle-timeout 10m --retention-period 1h)" >> "$GITHUB_ENV"
         env:
           GH_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}
           HEAD_REF: ${{ github.head_ref }}
+      - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
+        with:
+          node-version: 20
+      - run: npm ci
+      - run: npx playwright install --with-deps
+      - name: Wait for Codespace to be active
+        run: |
+          set -Eeuo pipefail
+
+          MAX_WAIT_SECONDS=$((3 * 60))
+          SECONDS_ELAPSED=0
+
+          while true; do
+            STATE=$(gh codespace list --json name,state --jq ".[] | select(.name == \"$CODESPACE_NAME\") | .state")
+            echo "Current state: $STATE"
+            if [ "$STATE" == "Available" ]; then
+              echo "Codespace is active!"
+              break
+            fi
+            if [ $SECONDS_ELAPSED -ge $MAX_WAIT_SECONDS ]; then
+              echo "Timeout reached. Codespace is not active."
+              exit 1
+            fi
+            sleep 5
+            SECONDS_ELAPSED=$((SECONDS_ELAPSED + 5))
+          done
+        env:
+          GH_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}
       - run: cd test/${{ inputs.flavor }}/features && npm test
         env:
           GITHUB_USER: ${{ secrets.TEST_GITHUB_USER }}
diff --git a/test/cpp/features/pages/codespace.pom.ts b/test/cpp/features/pages/codespace.pom.ts
index 4bcce96a..59235999 100644
--- a/test/cpp/features/pages/codespace.pom.ts
+++ b/test/cpp/features/pages/codespace.pom.ts
@@ -41,7 +41,7 @@ export class CodespacePage {
     test.setTimeout(3 * 60 * 1000);
 
     for (const plugin of extensions) {
-      await expect(this.page.getByRole('tab', { name: plugin }).locator('a')).toBeVisible({ timeout: 5 * 60 * 1000 });
+      await expect(this.page.getByRole('tab', { name: plugin, exact: true }).locator('a')).toBeVisible({ timeout: 5 * 60 * 1000 });
     }
 
     await expect(this.page.getByRole('button', { name: 'Activating Extensions...' })).toBeHidden();
@@ -148,6 +148,10 @@ export class CodespacePage {
     await expect(this.page.getByRole('code')).toContainText(expected);
   }
 
+  async expectOutputToContain(expectedOutput: string) {
+    await expect(this.outputPanel).toContainText(expectedOutput, { timeout: 5 * 60 * 1000 });
+  }
+
   async expectFileContentsToMatch(actual: string, expected: string) {
     await this.executeInTerminal(`diff -s ${actual} ${expected}`);
     await expect(this.page.locator('#terminal')).toContainText(`Files ${actual} and ${expected} are identical`);
diff --git a/test/cpp/features/steps/codespace.steps.ts b/test/cpp/features/steps/codespace.steps.ts
index 52726326..674dcf4a 100644
--- a/test/cpp/features/steps/codespace.steps.ts
+++ b/test/cpp/features/steps/codespace.steps.ts
@@ -1,4 +1,3 @@
-import { expect } from "@playwright/test";
 import { Given, When, Then } from "./fixtures";
 
 Given("build configuration {string} is selected", async ({ codespacePage }, configuration: string) => {
@@ -26,7 +25,7 @@ When("the active document is saved", async ({ codespacePage }) => {
 });
 
 Then("the output should contain {string}", async ({ codespacePage }, expectedOutput: string) => {
-  await expect(codespacePage.outputPanel).toContainText(expectedOutput, { timeout: 5 * 60 * 1000 });
+  await codespacePage.expectOutputToContain(expectedOutput);
 });
 
 Then("the editor should contain {string}", async ({ codespacePage }, expectedContent: string) => {