ci.yaml

name: CI

# Controls when the workflow will run
on:
  # Triggers the workflow on push or pull request events.
  push: {}
  pull_request: {}

  # Run the pipeline daily so that we get continuous dogfooding.
  schedule:
    # Run at 8am UTC/midnight Pacific
    - cron: 0 8 * * *

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch: {}

jobs:
  check:
    name: Build
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v3
        with:
          node-version: "16"
          cache: yarn

      - id: install
        env:
          CYPRESS_INSTALL_BINARY: "0"
        run: yarn install --immutable

      - run: yarn build

      - run: yarn workspace @unflakable/cypress-plugin pack
      - uses: actions/upload-artifact@v3
        with:
          name: cypress-plugin
          path: packages/cypress-plugin/package.tgz

      - run: yarn workspace @unflakable/jest-plugin pack
      - uses: actions/upload-artifact@v3
        with:
          name: jest-plugin
          path: packages/jest-plugin/package.tgz

      - run: yarn workspace @unflakable/js-api pack
      - uses: actions/upload-artifact@v3
        with:
          name: js-api
          path: packages/js-api/package.tgz

      - name: Run @unflakable/cypress-plugin unit tests
        env:
          FORCE_COLOR: "1"
        run: yarn workspace @unflakable/cypress-plugin test

      - name: Run @unflakable/jest-plugin unit tests
        env:
          FORCE_COLOR: "1"
        run: yarn workspace @unflakable/jest-plugin test

      - if: ${{ always() && steps.install.outcome == 'success' }}
        run: yarn lint

      - if: ${{ always() && steps.install.outcome == 'success' }}
        run: yarn prettier:check

      # Don't block the build on audit failures except in the private fork. We still have Dependabot
      # alerts enabled in the public repo, however.
      - if: ${{ always() && github.repository == 'unflakable-javascript-private' && steps.install.outcome == 'success' }}
        run: yarn audit

      - uses: dorny/paths-filter@v2
        id: affects_plugins
        with:
          # Treat all branches as long-lived so that the diff is based on the previous push on the
          # current branch, rather than the default branch. This will print a warning on PRs since
          # it's ignored for PR events, but that's fine.
          base: ${{ github.ref }}
          filters: |
            cypress:
              - package.json
              - tsconfig.json
              - .github/workflows/ci.yaml
              - packages/cypress-plugin/**
              - packages/js-api/**
              - packages/plugins-common/**
            jest:
              - package.json
              - tsconfig.json
              - .github/workflows/ci.yaml
              - packages/jest-plugin/**
              - packages/js-api/**
              - packages/plugins-common/**
              - scripts/set-jest-version.ts
    outputs:
      affects_cypress: ${{ steps.affects_plugins.outputs.cypress }}
      affects_jest: ${{ steps.affects_plugins.outputs.jest }}

  cypress_linux_integration_tests:
    name: "Cypress ${{ matrix.cypress }} Linux Node ${{ matrix.node }} Integration Tests"
    runs-on: ubuntu-latest
    timeout-minutes: 80
    needs:
      # Don't incur the cost of the test matrix if the basic build fails.
      - check
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.check.outputs.affects_cypress == 'true'
    strategy:
      fail-fast: false
      max-parallel: 16
      matrix:
        node:
          - 16
          - 18
          - 20
        cypress:
          # FIXME: support earlier versions
          #- "10.0"
          #- "10.11"
          #- "11.0"
          - "11.2"
          - "12.0"
          - "12.10"
          - "12.14"
          - "12.15"
          - "12.16"
          - "12.17"
          - "13.0"
          - "13.1"
          - "13.2"
          # 13.3.2 introduced a regression that made that version (and 13.3.3) too flaky to test.
          # See:
          #   - https://github.com/cypress-io/cypress/issues/28141
          #   - https://github.com/cypress-io/cypress/issues/28148
          #- "13.3"
          - "13.4"
          - "13.5"
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node }}
          cache: yarn

      - name: Cache Cypress binary
        uses: actions/cache@v3
        with:
          path: ~/.cache/Cypress
          key: cypress-${{ runner.os }}-node${{ matrix.node }}-cypress${{ matrix.cypress }}

      - id: install
        env:
          CYPRESS_INSTALL_BINARY: "0"
        run: yarn install --immutable

      - uses: actions/download-artifact@v3
        with:
          path: .artifacts

      - name: Install pre-built plugin packages
        env:
          CYPRESS_INSTALL_BINARY: "0"
        run: |
          curl -Lo jq https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64
          chmod +x jq
          cat package.json \
            | ./jq '. + {"resolutions": (.resolutions + {
                "@unflakable/cypress-plugin": "file:./.artifacts/cypress-plugin/package.tgz",
                "@unflakable/jest-plugin": "file:./.artifacts/jest-plugin/package.tgz",
                "@unflakable/js-api": "file:./.artifacts/js-api/package.tgz"
              })}' > package-new.json
          mv package-new.json package.json
          yarn install --no-immutable

      - name: Build test dependencies
        run: yarn build:plugins-common && yarn build:test-common && yarn build:cypress-tests

      - name: Set Cypress version
        env:
          CYPRESS_INSTALL_BINARY: "0"
        run: |
          yarn set resolution "cypress@npm:11.2 - 13" ${{ matrix.cypress }}
          grep --after-context=1 "^\".*cypress.*" yarn.lock

      - name: Install Cypress binary
        run: yarn workspace cypress-integration exec cypress install

      - name: Test
        env:
          # Enable debug logs within the Jest tests that run Cypress. WARNING: these are very
          # verbose but are useful for seeing the raw chalk terminal codes.
          # DEBUG: unflakable:*

          # Enable debug logs within the Cypress plugin.
          TEST_DEBUG: "unflakable:*,cypress:server:*,cypress-verbose:server:browsers:cri-client:*"

          # Enable terminal colors for debug() output.
          DEBUG_COLORS: "1"

          # Make chalk emit TTY colors.
          FORCE_COLOR: "1"
        run: |
          if [ "${{ github.repository }}" == "unflakable/unflakable-javascript" ]; then
            export UNFLAKABLE_SUITE_ID=2QwtGckRudLNUGBsdkVEoSknck1
          else
            export UNFLAKABLE_SUITE_ID=2Qwt9RyPIbOI95C6qjXCzcTelni
          fi
          UNFLAKABLE_API_KEY=${{ secrets.UNFLAKABLE_API_KEY }} \
            yarn workspace cypress-integration test \
              --reporters @unflakable/jest-plugin/dist/reporter \
              --runner @unflakable/jest-plugin/dist/runner \
              --testRunner @unflakable/jest-plugin/dist/test-runner

  cypress_windows_integration_tests:
    name: "Cypress ${{ matrix.cypress }} Windows Node ${{ matrix.node }} Integration Tests"
    runs-on: windows-2019
    # Cypress on Windows is slowwww...
    timeout-minutes: 180
    needs:
      # Don't incur the cost of the test matrix if the basic build fails.
      - check
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.check.outputs.affects_cypress == 'true'
    strategy:
      fail-fast: false
      max-parallel: 16
      matrix:
        node:
          - 16
          - 18
          - 20
        cypress:
          # FIXME: support earlier versions
          #- "10.0"
          #- "10.11"
          #- "11.0"
          - "11.2"
          - "12.0"
          - "12.10"
          - "12.14"
          - "12.15"
          - "12.16"
          - "12.17"
          - "13.0"
          - "13.1"
          - "13.2"
          #- "13.3"
          - "13.4"
          - "13.5"
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node }}
          cache: yarn

      - name: Get AppData directory
        run: echo ("LOCALAPPDATA=" + $env:LOCALAPPDATA) >> $env:GITHUB_ENV

      - name: Cache Cypress binary
        uses: actions/cache@v3
        with:
          path: ${{ env.LOCALAPPDATA }}\Cypress\Cache
          key: cypress-${{ runner.os }}-node${{ matrix.node }}-cypress${{ matrix.cypress }}

      - id: install
        env:
          CYPRESS_INSTALL_BINARY: "0"
        run: |
          yarn install --immutable

      - uses: actions/download-artifact@v3
        with:
          path: .artifacts

      - name: Install pre-built plugin packages
        env:
          CYPRESS_INSTALL_BINARY: "0"
        shell: bash
        run: |
          curl -Lo jq.exe https://github.com/jqlang/jq/releases/download/jq-1.6/jq-win64.exe
          cat package.json \
            | ./jq.exe '. + {"resolutions": (.resolutions + {
                "@unflakable/cypress-plugin": "file:./.artifacts/cypress-plugin/package.tgz",
                "@unflakable/jest-plugin": "file:./.artifacts/jest-plugin/package.tgz",
                "@unflakable/js-api": "file:./.artifacts/js-api/package.tgz"
              })}' > package-new.json
          mv package-new.json package.json
          yarn install --no-immutable

      - name: Build test dependencies
        run: yarn build:plugins-common && yarn build:test-common && yarn build:cypress-tests

      - name: Set Cypress version
        env:
          CYPRESS_INSTALL_BINARY: "0"
        run: |
          yarn set resolution "cypress@npm:11.2 - 13" ${{ matrix.cypress }}
          Select-String -Pattern '^".*cypress.*' -Path yarn.lock -Context 0,1

      - name: Install Cypress binary
        run: yarn workspace cypress-integration exec cypress install

      - name: Test
        env:
          # Enable debug logs within the Jest tests that run Cypress. WARNING: these are very
          # verbose but are useful for seeing the raw chalk terminal codes.
          # DEBUG: unflakable:*

          # Enable debug logs within the Cypress plugin.
          TEST_DEBUG: "unflakable:*,cypress:server:*,cypress-verbose:server:browsers:cri-client:*"

          # Enable terminal colors for debug() output.
          DEBUG_COLORS: "1"

          # Make chalk emit TTY colors.
          FORCE_COLOR: "1"

          UNFLAKABLE_API_KEY: ${{ secrets.UNFLAKABLE_API_KEY }}
          UNFLAKABLE_SUITE_ID: ${{ github.repository == 'unflakable/unflakable-javascript' && '2QwtGckRudLNUGBsdkVEoSknck1' || '2Qwt9RyPIbOI95C6qjXCzcTelni' }}
        run: |
          yarn workspace cypress-integration test `
            --reporters @unflakable/jest-plugin/dist/reporter `
            --runner @unflakable/jest-plugin/dist/runner `
            --testRunner @unflakable/jest-plugin/dist/test-runner

  jest_linux_integration_tests:
    name: "Jest ${{ matrix.jest }} Linux Node ${{ matrix.node }} Integration Tests"
    runs-on: ubuntu-latest
    timeout-minutes: 20
    needs:
      # Don't incur the cost of the test matrix if the basic build fails.
      - check
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.check.outputs.affects_jest == 'true'
    strategy:
      fail-fast: false
      max-parallel: 12
      matrix:
        node:
          - 16
          - 18
          - 20
        jest:
          - "29.7"
          - "29.6"
          - "29.5"
          - "29.4"
          - "29.3"
          - "29.2"
          - "29.1"
          - "29.0"
          - "28.1"
          - "28.0"
          - "27.5"
          - "27.4"
          - "27.3"
          - "27.2"
          - "27.1"
          - "27.0"
          - "26.6"
          - "26.5"
          - "26.4"
          - "26.3"
          - "26.2"
          - "26.1"
          - "26.0"
          - "25.5"
          - "25.4"
          - "25.3"
          - "25.2"
          - "25.1"
    env:
      CYPRESS_INSTALL_BINARY: "0"

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node }}
          cache: yarn

      - id: install
        run: yarn install --immutable

      - uses: actions/download-artifact@v3
        with:
          path: .artifacts

      - name: Install pre-built plugin packages
        run: |
          curl -Lo jq https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64
          chmod +x jq
          cat package.json \
            | ./jq '. + {"resolutions": (.resolutions + {
                "@unflakable/jest-plugin": "file:./.artifacts/jest-plugin/package.tgz",
                "@unflakable/js-api": "file:./.artifacts/js-api/package.tgz"
              })}' > package-new.json
          mv package-new.json package.json
          yarn install --no-immutable

      # Since Jest is composed of many packages, we need to make sure that they're all using the
      # same major and minor versions. Otherwise, NPM will keep the newer minor versions since
      # they're semver-compatible. This script iteratively installs the lowest compatible semver
      # version for each package until yarn.lock converges. See the script for some edge cases due
      # to missing package versions with certain minor version numbers.
      - name: Set Jest version
        run: |
          yarn set-jest-version ${{ matrix.jest }}
          grep --after-context=1 "^\".*jest.*" yarn.lock

      - name: Set resolution for chalk to 3.0.0
        if: ${{ startsWith(matrix.jest, '25.')  }}
        run: yarn set resolution "chalk@npm:^3.0.0 || ^4.0.0" 3.0

      - name: Build test dependencies
        run: yarn build:plugins-common && yarn build:test-common

      - name: Test
        env:
          # Enable debug logs within the jest-plugin/test/integration Jest tests that invoke Jest
          # on the jest-plugin/test/integration-input test cases. WARNING: these are very verbose
          # but are useful for seeing the raw chalk terminal codes.
          # DEBUG: unflakable:*

          # Enable debug logs within the Jest plugin.
          TEST_DEBUG: "unflakable:*"

          # Enable terminal colors for debug() output.
          DEBUG_COLORS: "1"

          # Make chalk emit TTY colors.
          FORCE_COLOR: "1"
        run: |
          if [ "${{ github.repository }}" == "unflakable/unflakable-javascript" ]; then
            export UNFLAKABLE_SUITE_ID=29KWCuK12VnU7pkpvWgrGS0woAX
          else
            export UNFLAKABLE_SUITE_ID=28UidZ8cSKjRe4g1xkd9EE8noDF
          fi
          UNFLAKABLE_API_KEY=${{ secrets.UNFLAKABLE_API_KEY }} \
            yarn workspace jest-integration test \
            --reporters @unflakable/jest-plugin/dist/reporter \
            --runner @unflakable/jest-plugin/dist/runner

  jest_windows_integration_tests:
    name: "Jest ${{ matrix.jest }} Windows Node ${{ matrix.node }} Integration Tests"
    runs-on: windows-2019
    timeout-minutes: 30
    needs:
      # Don't incur the cost of the test matrix if the basic build fails.
      - check
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.check.outputs.affects_jest == 'true'
    strategy:
      fail-fast: false
      max-parallel: 12
      matrix:
        node:
          - 16
          - 18
          - 20
        jest:
          - "29.7"
          - "29.6"
          - "29.5"
          - "29.4"
          - "29.3"
          - "29.2"
          - "29.1"
          - "29.0"
          - "28.1"
          - "28.0"
          - "27.5"
          - "27.4"
          - "27.3"
          - "27.2"
          - "27.1"
          - "27.0"
          - "26.6"
          - "26.5"
          - "26.4"
          - "26.3"
          - "26.2"
          - "26.1"
          - "26.0"
          - "25.5"
          - "25.4"
          - "25.3"
          - "25.2"
          - "25.1"
    env:
      CYPRESS_INSTALL_BINARY: "0"

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node }}
          cache: yarn

      - id: install
        run: yarn install --immutable

      - uses: actions/download-artifact@v3
        with:
          path: .artifacts

      - name: Install pre-built plugin packages
        shell: bash
        run: |
          curl -Lo jq.exe https://github.com/jqlang/jq/releases/download/jq-1.6/jq-win64.exe
          cat package.json \
            | ./jq.exe '. + {"resolutions": (.resolutions + {
                "@unflakable/jest-plugin": "file:./.artifacts/jest-plugin/package.tgz",
                "@unflakable/js-api": "file:./.artifacts/js-api/package.tgz"
              })}' > package-new.json
          mv package-new.json package.json
          yarn install --no-immutable

      - name: Set Jest version
        run: |
          yarn set-jest-version ${{ matrix.jest }}
          Select-String -Pattern '^".*jest.*' -Path yarn.lock -Context 0,1

      - name: Set resolution for chalk to 3.0.0
        if: ${{ startsWith(matrix.jest, '25.')  }}
        run: yarn set resolution "chalk@npm:^3.0.0 || ^4.0.0" 3.0

      - name: Build test dependencies
        run: yarn build:plugins-common && yarn build:test-common

      - name: Test
        env:
          # Enable debug logs within the jest-plugin/test/integration Jest tests that invoke Jest
          # on the jest-plugin/test/integration-input test cases. WARNING: these are very verbose
          # but are useful for seeing the raw chalk terminal codes.
          # DEBUG: unflakable:*

          # Enable debug logs within the Jest plugin.
          TEST_DEBUG: "unflakable:*"

          # Enable terminal colors for debug() output.
          DEBUG_COLORS: "1"

          # Make chalk emit TTY colors.
          FORCE_COLOR: "1"

          UNFLAKABLE_API_KEY: ${{ secrets.UNFLAKABLE_API_KEY }}
          UNFLAKABLE_SUITE_ID: ${{ github.repository == 'unflakable/unflakable-javascript' && '29KWCuK12VnU7pkpvWgrGS0woAX' || '28UidZ8cSKjRe4g1xkd9EE8noDF' }}
        run: |
          yarn workspace jest-integration test `
            --reporters @unflakable/jest-plugin/dist/reporter `
            --runner @unflakable/jest-plugin/dist/runner