chore(tests): add stylesheet fixture to test for regression from docs…

… bundle (#1429)

.github/workflows/development.yml

@@ -133,7 +133,7 @@ jobs:
         run: pnpm exec playwright install --with-deps
 
       - name: Run Playwright tests
-        run: pnpm exec playwright test playwright/global.spec.ts --workers 6
+        run: pnpm exec playwright test playwright/smoke --workers 6
 
       - uses: actions/upload-artifact@v4
         if: always()

.github/workflows/playwright.yml

@@ -1,7 +1,5 @@
 name: Playwright Tests
-on:
-  push:
-    branches: [main]
+on: pull_request
 
 env:
   TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
@@ -11,25 +9,43 @@ env:
   WORKOS_CLIENT_ID: ${{ secrets.WORKOS_CLIENT_ID }}
   # HUME_API_KEY: ${{ secrets.HUME_API_KEY }}
 
-jobs: {}
-  # test:
-  #   timeout-minutes: 60
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v4
+jobs:
+  test:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
 
-  #     - name: Install
-  #       uses: ./.github/actions/install
+      - name: Install
+        uses: ./.github/actions/install
 
-  #     - name: Install Playwright Browsers
-  #       run: pnpm exec playwright install --with-deps
+      - name: Compile and build
+        run: pnpm turbo compile codegen build
+        env:
+          FERN_TOKEN: ${{ secrets.FERN_TOKEN }}
+          WORKOS_API_KEY: ${{ secrets.WORKOS_API_KEY }}
+          WORKOS_CLIENT_ID: ${{ secrets.WORKOS_CLIENT_ID }}
 
-  #     - name: Run Playwright tests
-  #       run: pnpm exec playwright test --debug
+      - name: Build next bundle
+        run: cd packages/ui/local-preview-bundle; pnpm next build
 
-  #     - uses: actions/upload-artifact@v4
-  #       if: always()
-  #       with:
-  #         name: playwright-report
-  #         path: playwright-report/
-  #         retention-days: 30
+      - name: Install Playwright Browsers
+        run: pnpm exec playwright install --with-deps
+
+      - name: Install fern-dev API
+        env:
+          NPM_TOKEN: ${{ secrets.FERN_NPM_TOKEN }}
+          FERN_TOKEN: ${{ secrets.FERN_ORG_TOKEN_DEV }}
+        run: |
+          npm config set //registry.npmjs.org/:_authToken $NPM_TOKEN
+          npm install -g @fern-api/fern-api-dev
+
+      - name: Run Playwright tests
+        run: pnpm exec playwright test playwright/fixtures
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30

.github/workflows/preview.yml

@@ -131,10 +131,7 @@ jobs:
         run: pnpm exec playwright install --with-deps
 
       - name: Run Playwright tests
-        run: pnpm exec playwright test playwright/global.spec.ts --workers 6
-
-      - name: Run Playwright tests
-        run: pnpm exec playwright test playwright/versioned-docs.spec.ts
+        run: pnpm exec playwright test playwright/smoke --workers 6
 
       - uses: actions/upload-artifact@v4
         if: always()

playwright/fixtures/stylesheet/fern/docs.yml

@@ -0,0 +1,9 @@
+instances:
+  - url: https://stylesheet-fixture.docs.buildwithfern.com
+title: Sample Styled Docs
+navigation:
+  - page: Test Page
+    path: ./page.mdx
+colors:
+  accentPrimary: "#ffffff"
+  background: "#330000"

playwright/fixtures/stylesheet/fern/fern.config.json

@@ -0,0 +1,4 @@
+{
+  "organization": "stylesheet-fixture",
+  "version": "*"
+}

playwright/fixtures/stylesheet/fern/page.mdx

@@ -0,0 +1,4 @@
+---
+title: Welcome to a demo site
+slug: page
+---

playwright/fixtures/stylesheet/stylesheet.spec.ts

@@ -0,0 +1,18 @@
+import test, { expect } from "@playwright/test";
+import { runFixture } from "../../test-runner";
+
+const port = "6983";
+test("Check CSS variable values", async ({ page }) => {
+    const computedStyles = await runFixture("stylesheet", port, async () => {
+        await page.goto(`localhost:${port}`);
+
+        const element = page.locator("body");
+        return await element.evaluate((el) => {
+            const styles = window.getComputedStyle(el);
+            return {
+                backgroundColor: styles.getPropertyValue("--background"),
+            };
+        });
+    });
+    expect(computedStyles.backgroundColor.trim()).toBe("51, 0, 0");
+});

playwright/global-inclusions.yml → playwright/inclusions.yml

@@ -1,4 +1,4 @@
-global-inclusions:
+existence:
   - docs.trykeet.com
   - docs.argolabs.ai
   - docs.conductorquantum.com
@@ -45,7 +45,7 @@ global-inclusions:
   - dev.documentation.sayari.com
   - fern.assemblyai.com
 
-favicon-inclusions:
+favicon:
   - docs.argolabs.ai
   - docs.conductorquantum.com
   - developers.gappify.com
@@ -57,14 +57,10 @@ favicon-inclusions:
   - www.intern.mavenagi.com
   - docs.schematichq.com
   - plantstore.dev
-  - www.plantstore.dev
   - docs.boundaryml.com
-  - docs.qlty.ai
   - docs.rightbrain.ai
   - docs.getkard.com
   - docs.syndicate.io
-  - docs.qlty.us
-  - hume.ferndocs.com
   - docs.stack-auth.com
   - gemini2.ferndocs.com
   - docs.darwincloud.xyz
@@ -77,7 +73,6 @@ favicon-inclusions:
   - docs.reka.ai
   - docs.aiaplatform.com.au
   - docs.anterior.com
-  - developers.beehiiv.com
   - docs.scorecard.io
   - fern.humanloop.com
   - docs.goldenbasis.com

playwright/run-fixture.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd playwright/fixtures/$1
+fern-dev docs dev --port $2 --bundle-path ../../../packages/ui/local-preview-bundle/out

playwright/smoke/existence.spec.ts

@@ -0,0 +1,10 @@
+import { expect, test } from "@playwright/test";
+import { getPlaywrightTestUrls } from "../test-runner";
+
+const existenceUrls = getPlaywrightTestUrls("existence");
+existenceUrls.forEach((testUrl) => {
+    test(`Check if ${testUrl} is online`, async ({ page }) => {
+        const response = await page.goto(testUrl);
+        expect(response?.status()).toBe(200);
+    });
+});

playwright/smoke/favicon.spec.ts

@@ -0,0 +1,22 @@
+import test, { expect } from "@playwright/test";
+import { getPlaywrightTestUrls } from "../test-runner";
+
+const faviconUrls = getPlaywrightTestUrls("favicon");
+faviconUrls.forEach((testUrl) => {
+    test(`Check if favicon exists and URL does not return 404 for ${testUrl}`, async ({ page }) => {
+        await page.goto(testUrl);
+
+        const faviconUrl = await page.getAttribute('link[rel="icon"]', "href", {
+            timeout: 5000,
+        });
+
+        expect(faviconUrl).not.toBeNull();
+
+        if (faviconUrl) {
+            const response = await page.goto(faviconUrl);
+            expect(response?.status()).toBe(200);
+        } else {
+            throw new Error("Favicon link not found");
+        }
+    });
+});

playwright/versioned-docs.spec.ts → playwright/smoke/versioned-docs.spec.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "@playwright/test";
-import { generatePreviewContext } from "./utils";
+import { generatePreviewContext } from "../utils";
 
 /**
  * This test will run on a sample of urls from customer docs using versions.

playwright/test-runner.ts

@@ -0,0 +1,67 @@
+import { spawn } from "child_process";
+import * as fs from "fs";
+import * as yaml from "js-yaml";
+
+export function getPlaywrightTestUrls(type: string): string[] {
+    const testUrls: string[] = [];
+    const playwrightConfig = yaml.load(fs.readFileSync("playwright/inclusions.yml", "utf-8"));
+
+    if (!(type in playwrightConfig)) {
+        throw new Error(`Test type ${type} not found in playwright/inclusions.yml`);
+    }
+    const testInclusions = new Set<string>(playwrightConfig[type]);
+
+    function processLineByLineSync(filePath: string): void {
+        const fileContent = fs.readFileSync(filePath, "utf-8");
+
+        const lines = fileContent.split(/\r?\n/);
+
+        lines.forEach((line) => {
+            const urlPattern = /\(([^)]+\?host=([^&]+))\)/;
+            const match = line.match(urlPattern);
+            if (match) {
+                const fullUrl = match[1];
+                const isIncludedUrl = match[2];
+                if (fullUrl && testInclusions.has(isIncludedUrl ?? "")) {
+                    testUrls.push(fullUrl);
+                }
+            }
+        });
+    }
+
+    processLineByLineSync("preview.txt");
+
+    if (testUrls.length === 0) {
+        throw new Error("No URLs found in preview.txt");
+    }
+
+    return testUrls;
+}
+
+function sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function runFixture<T>(fixturePath: string, port: string, test: () => Promise<T>): Promise<T> {
+    const l = spawn("playwright/run-fixture.sh", [fixturePath, port]);
+    l.stdout.pipe(process.stdout);
+    l.stderr.pipe(process.stderr);
+    await sleep(7500);
+
+    const result = await test();
+
+    await sleep(500);
+    const livePort = spawn("lsof", ["-i", `:${port}`, "-t"]);
+    livePort.stdout.on("data", (data) => {
+        data.toString()
+            .split("\n")
+            .forEach((pid: string) => {
+                if (pid.length > 0 && Number(pid) !== process.pid) {
+                    process.kill(Number(pid));
+                }
+            });
+    });
+    await sleep(500);
+
+    return result;
+}