feat: playwright tests (#136)

* feat: playwright tests

* chore: commit package.lock

* fix: template tests

* chore: don't spam on telegram

.github/workflows/playwright.yml

@@ -0,0 +1,17 @@
+name: E2E Tests
+
+on:
+  deployment_status:
+jobs:
+  run-e2es:
+    if: github.event_name == 'deployment_status' && github.event.deployment_status.state == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: npm ci && npx playwright install --with-deps
+      - name: Run tests
+        run: npx playwright test
+        env:
+          BASE_URL: ${{ github.event.deployment_status.environment_url }}
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}

.gitignore

@@ -185,7 +185,12 @@ bun.lockb
 *.Dockerfile
 docker-compose*
 
-package-lock.json
 
 entrypoint.sh
-packages/frontend/Frontend.code-workspace
+
+/test-results/
+/playwright-report/
+/playwright/
+/blob-report/
+/playwright/.cache/
+/e2e/.auth/

e2e/auth.setup.ts

@@ -0,0 +1,53 @@
+import { test, expect } from "@playwright/test"
+
+import { deleteOrg } from "./db-utils"
+
+const authFile = "e2e/.auth/user.json"
+
+test.beforeAll(async () => {
+  // Sometimes the teardown isn't called, so we need to clean up the database before running the tests
+  await deleteOrg()
+})
+
+test("signup flow", async ({ page }) => {
+  await page.goto("/")
+
+  await page.getByRole("link", { name: "Sign Up" }).click()
+
+  await page.waitForURL("**/signup")
+
+  await page.getByPlaceholder("Your email").click()
+  await page.getByPlaceholder("Your email").fill("[email protected]")
+
+  await page.getByPlaceholder("Your full name").click()
+  await page.getByPlaceholder("Your full name").fill("test test")
+
+  await page.getByPlaceholder("Your password").click()
+  await page.getByPlaceholder("Your password").fill("testtest")
+
+  await page.getByRole("button", { name: "Continue →" }).click()
+
+  await page.getByPlaceholder("Your project name").click()
+  await page.getByPlaceholder("Your project name").fill("TESTPROJECT")
+
+  await page.getByPlaceholder("Organization name").click()
+  await page.getByPlaceholder("Organization name").fill("TESTORG")
+
+  await page.getByLabel("-49").check()
+
+  await page.getByRole("button", { name: "Create account" }).click()
+
+  await page.waitForNavigation()
+
+  await expect(page.getByText("Are you free in the next days")).toBeVisible()
+
+  await page.getByRole("button", { name: "Skip to Dashboard" }).click()
+
+  await page.waitForURL("**/analytics")
+
+  await expect(
+    page.getByRole("heading", { name: "Waiting for data..." }),
+  ).toBeVisible()
+
+  await page.context().storageState({ path: authFile })
+})

e2e/db-utils.ts

@@ -0,0 +1,13 @@
+import sql from "../packages/backend/src/utils/db"
+
+export async function setOrgPro() {
+  await sql`update org set plan = 'pro' where name = 'TESTORG'`
+}
+
+export async function setOrgFree() {
+  await sql`update org set plan = 'free' where name = 'TESTORG'`
+}
+
+export async function deleteOrg() {
+  await sql`delete from org where name = 'TESTORG'`
+}

e2e/global.teardown.ts

@@ -0,0 +1,6 @@
+import { test } from "@playwright/test"
+import { deleteOrg } from "./db-utils"
+
+test("clean up database", async ({}) => {
+  await deleteOrg()
+})

e2e/login.spec.ts

@@ -0,0 +1,22 @@
+import { test, expect } from "@playwright/test"
+
+test("logout and login flow", async ({ page }) => {
+  await page.goto("/")
+
+  // logout
+  await page.getByTestId("account-sidebar-item").click()
+  await page.getByTestId("logout-button").click()
+
+  await page.waitForURL("**/login")
+
+  // log back in
+  await page.getByPlaceholder("Your email").click()
+  await page.getByPlaceholder("Your email").fill("[email protected]")
+
+  await page.getByPlaceholder("Your password").click()
+  await page.getByPlaceholder("Your password").fill("testtest")
+
+  await page.getByRole("button", { name: "Login" }).click()
+
+  await page.waitForURL("**/analytics")
+})

e2e/projects.spec.ts

@@ -0,0 +1,30 @@
+import { test, expect } from "@playwright/test"
+import { setOrgFree, setOrgPro } from "./db-utils"
+
+test.beforeAll(async () => {
+  // need to be pro for multiple projects
+  await setOrgPro()
+})
+
+test.afterAll(async () => {
+  await setOrgFree()
+})
+
+test("create new project and rename it", async ({ page }) => {
+  await page.goto("/")
+
+  //wait for requests to finish
+  await page.waitForLoadState("networkidle")
+
+  await page.getByRole("button", { name: "TESTPROJECT" }).click()
+
+  await page.getByTestId("new-project").click()
+
+  await page.getByRole("heading", { name: "Project #" }).click()
+  await page.getByTestId("rename-input").fill("TESTPROJECT2")
+  await page.getByTestId("rename-input").press("Enter")
+
+  await expect(
+    page.getByRole("heading", { name: "TESTPROJECT2" }),
+  ).toBeVisible()
+})

e2e/templates.spec.ts

@@ -0,0 +1,54 @@
+import { test, expect } from "@playwright/test"
+import { setOrgFree, setOrgPro } from "./db-utils"
+
+test.beforeAll(async () => {
+  await setOrgPro()
+})
+
+test.afterAll(async () => {
+  await setOrgFree()
+})
+
+// run tests one after another
+test.describe.configure({ mode: "serial" })
+
+test("create new template and test basic playground", async ({ page }) => {
+  await page.goto("/prompts")
+
+  await page.waitForLoadState("networkidle")
+
+  await page.getByTestId("empty-action").click()
+
+  await page.getByTestId("rename-template-input").fill("first-template")
+  await page.getByTestId("rename-template-input").press("Enter")
+
+  await expect(page.getByText("first-template")).toBeVisible()
+
+  await page.getByTestId("run-playground").click()
+
+  // assistant's response
+  await expect(page.getByText("Hello!")).toBeVisible()
+  await expect(page.locator("#HERE").getByText("assistant")).toBeVisible()
+})
+
+test("create new template and deploy", async ({ page }) => {
+  await page.goto("/prompts")
+
+  await page.waitForLoadState("networkidle")
+
+  await page.getByTestId("create-template").click()
+
+  await page.getByTestId("rename-template-input").fill("test-template")
+  await page.getByTestId("rename-template-input").press("Enter")
+
+  await page.getByTestId("deploy-template").click()
+
+  await expect(page.getByText("Template deployed")).toBeVisible()
+
+  await page.getByText("Hi!").click()
+  await page.getByText("Hi!").fill("This is another test")
+
+  await page.getByTestId("save-template").click()
+
+  await expect(page.locator("a").filter({ hasText: "v2" })).toBeVisible()
+})