conformance.yaml

name: Conformance tests

permissions: {}

on:
  pull_request:
    branches:
      - 'main'
      - 'release*'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  prepare-images:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: build-images
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
      - name: ko build
        shell: bash
        run: |
          set -e
          VERSION=${{ github.ref_name }} make docker-save-image-all
      - name: upload images archive
        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
        with:
          name: kyverno.tar
          path: kyverno.tar
          retention-days: 1
          if-no-files-found: error

  prepare-cli:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: build-cli
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
      - name: Build CLI
        shell: bash
        run: |
          set -e
          VERSION=${{ github.ref_name }} make build-cli
      - name: upload images archive
        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
        with:
          name: kubectl-kyverno
          path: cmd/cli/kubectl-kyverno/kubectl-kyverno
          retention-days: 1
          if-no-files-found: error

  # runs conformance test suites with configuration:
  standard:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: standard
            values:
              - standard
        k8s-version:
          - name: v1.25
            version: v1.25.11
          - name: v1.26
            version: v1.26.6
          - name: v1.27
            version: v1.27.3
          - name: v1.28
            version: v1.28.0
        tests:
          - autogen
          - background-only
          - cleanup
          - deferred
          - events
          - exceptions
          - filter
          - generate/clusterpolicy
          - generate/policy
          - generate/validation
          - mutate
          - policy-validation
          - rangeoperators
          - rbac
          - reports
          - validate
          - verify-manifests
          - verifyImages
          - webhooks
    needs: prepare-images
    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
      - name: Create kind cluster
        shell: bash
        run: |
          set -e
          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        shell: bash
        run: |
          set -e
          make kind-load-image-archive
      - name: Install kyverno
        shell: bash
        run: |
          set -e
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Test with kuttl
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -e
          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} --config ./test/conformance/kuttl/_config/common.yaml
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs

  chainsaw:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: standard
            values:
              - standard
        k8s-version:
          - name: v1.25
            version: v1.25.11
          - name: v1.26
            version: v1.26.6
          - name: v1.27
            version: v1.27.3
          - name: v1.28
            version: v1.28.0
        tests:
          # - autogen
          - background-only
          # - cleanup
          # - deferred
          # - events
          # - exceptions
          # - filter
          # - generate/clusterpolicy
          # - generate/policy
          # - generate/validation
          # - mutate
          # - policy-validation
          - rangeoperators
          - rbac
          # - reports
          # - validate
          # - verify-manifests
          # - verifyImages
          - webhooks
    needs: prepare-images
    name: chainsaw - ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
      - name: Create kind cluster
        shell: bash
        run: |
          set -e
          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        shell: bash
        run: |
          set -e
          make kind-load-image-archive
      - name: Install kyverno
        shell: bash
        run: |
          set -e
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Install Chainsaw
        uses: kyverno/chainsaw/.github/actions/install@704abd5ea8fd74189e1192733a879a00a7d527f5 # main
        with:
          release: v0.0.6-alpha.1
      - name: Test with Chainsaw
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -e
          chainsaw test  --config ./test/conformance/chainsaw/_config/common.yaml --test-dir ./test/conformance/chainsaw/${{ matrix.tests }} --no-color=false
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs

  # runs conformance test suites with configuration:
  ttl:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: ttl
            values:
              - standard
              - ttl
        k8s-version:
          - name: v1.25
            version: v1.25.11
          - name: v1.26
            version: v1.26.6
          - name: v1.27
            version: v1.27.3
          - name: v1.28
            version: v1.28.0
        tests:
          - ttl
    needs: prepare-images
    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
      - name: Create kind cluster
        shell: bash
        run: |
          set -e
          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        shell: bash
        run: |
          set -e
          make kind-load-image-archive
      - name: Install kyverno
        shell: bash
        run: |
          set -e
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Test with kuttl
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -e
          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} --config ./test/conformance/kuttl/_config/common.yaml
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs

  # runs conformance test suites with configuration:
  force-failure-policy-ignore:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: force-failure-policy-ignore
            values:
              - standard
              - force-failure-policy-ignore
        k8s-version:
          - name: v1.25
            version: v1.25.11
          - name: v1.26
            version: v1.26.6
          - name: v1.27
            version: v1.27.3
          - name: v1.28
            version: v1.28.0
        tests:
          - force-failure-policy-ignore
          - rbac
    needs: prepare-images
    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
      - name: Create kind cluster
        shell: bash
        run: |
          set -e
          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        shell: bash
        run: |
          set -e
          make kind-load-image-archive
      - name: Install kyverno
        shell: bash
        run: |
          set -e
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Test with kuttl
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -e
          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} --config ./test/conformance/kuttl/_config/common.yaml
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs

  # runs conformance test suites with configuration:
  validating-admission-policies-v1alpha1:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: validating-admission-policies
            values:
              - standard
              - generate-validating-admission-policy
        k8s-version:
          - name: v1.27
            version: v1.27.3
        tests:
          - generate-validating-admission-policy
    needs: prepare-images
    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
      - name: Create kind cluster
        run: |
          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
          export KIND_CONFIG=vap-v1alpha1
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        run: make kind-load-image-archive
      - name: Install kyverno
        run: |
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Test with kuttl
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} \
            --config ./test/conformance/kuttl/_config/common.yaml
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs
  
  # runs conformance test suites with configuration:
  validating-admission-policies-v1beta1:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: validating-admission-policies
            values:
              - standard
              - generate-validating-admission-policy
        k8s-version:
          - name: v1.28
            version: v1.28.0
        tests:
          - generate-validating-admission-policy
    needs: prepare-images
    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
      - name: Create kind cluster
        run: |
          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
          export KIND_CONFIG=vap-v1beta1
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        run: make kind-load-image-archive
      - name: Install kyverno
        run: |
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Test with kuttl
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} \
            --config ./test/conformance/kuttl/_config/common.yaml
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs
  
  # runs conformance test suites with configuration:
  validating-admission-policy-reports-v1alpha1:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: validating-admission-policy-reports
            values:
              - standard
              - validating-admission-policy-reports
        k8s-version:
          - name: v1.26
            version: v1.26.6
          - name: v1.27
            version: v1.27.3
        tests:
          - validating-admission-policy-reports
    needs: prepare-images
    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
      - name: Create kind cluster
        run: |
          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
          export KIND_CONFIG=vap-v1alpha1
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        run: make kind-load-image-archive
      - name: Install kyverno
        run: |
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Test with kuttl
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} \
            --config ./test/conformance/kuttl/_config/common.yaml
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs
  
  # runs conformance test suites with configuration:
  validating-admission-policy-reports-v1beta1:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: validating-admission-policy-reports
            values:
              - standard
              - validating-admission-policy-reports
        k8s-version:
          - name: v1.28
            version: v1.28.0
        tests:
          - validating-admission-policy-reports
    needs: prepare-images
    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
      - name: Create kind cluster
        run: |
          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
          export KIND_CONFIG=vap-v1beta1
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        run: make kind-load-image-archive
      - name: Install kyverno
        run: |
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Test with kuttl
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} \
            --config ./test/conformance/kuttl/_config/common.yaml
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs

  # runs conformance test suites with configuration:
  custom-sigstore:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: custom-sigstore
            values:
              - standard
              - custom-sigstore
        k8s-version:
          - name: v1.25
            version: v1.25.x
          - name: v1.26
            version: v1.26.x
          - name: v1.27
            version: v1.27.x
          - name: v1.28
            version: v1.28.x
        tests:
          - custom-sigstore
    needs: prepare-images
    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
      - name: Create kind cluster and setup Sigstore Scaffolding
        uses: sigstore/scaffolding/actions/setup@b0c456efaf2843a1c4acea01aa9a8fee503412c0
        with:
          version: 'v0.6.8'
          k8s-version: ${{ matrix.k8s-version.version }}
          knative-version: '1.10.0'
      - name: Create TUF values config map
        run: |
          kubectl create namespace kyverno
          kubectl -n kyverno create configmap tufvalues --from-literal=TUF_MIRROR=$TUF_MIRROR --from-literal=FULCIO_URL=$FULCIO_URL --from-literal=REKOR_URL=$REKOR_URL --from-literal=CTLOG_URL=$CTLOG_URL --from-literal=ISSUER_URL=$ISSUER_URL
          kubectl -n tuf-system get secrets tuf-root -oyaml | sed 's/namespace: .*/namespace: kyverno/' | kubectl create -f -
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        shell: bash
        run: |
          set -e
          make kind-load-image-archive
      - name: Install kyverno
        shell: bash
        run: |
          set -e
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Install crane
        uses: imjasonh/setup-crane@00c9e93efa4e1138c9a7a5c594acd6c75a2fbf0c
      - name: Install Cosign
        uses: sigstore/cosign-installer@1fc5bd396d372bee37d608f955b336615edf79c8
      - name: Create test image
        shell: bash
        run: |
          DIGEST=$(crane digest cgr.dev/chainguard/static)   
          IMAGE_NAME=$(uuidgen | tr "[:upper:]" "[:lower:]")
          TEST_IMAGE_URL=ttl.sh/${IMAGE_NAME}:1h
          crane copy cgr.dev/chainguard/static@$DIGEST $TEST_IMAGE_URL
          cosign initialize --mirror $TUF_MIRROR --root $TUF_MIRROR/root.json
          COSIGN_EXPERIMENTAL=1 cosign sign --rekor-url $REKOR_URL --fulcio-url $FULCIO_URL $TEST_IMAGE_URL --identity-token `curl -s $ISSUER_URL` -y
          echo "TEST_IMAGE_URL=$TEST_IMAGE_URL" >> $GITHUB_ENV
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Test with kuttl
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -e
          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} --config ./test/conformance/kuttl/_config/common.yaml
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs

  # runs conformance test suites with configuration:
  default:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: default
            values:
              - default
        k8s-version:
          - name: v1.25
            version: v1.25.11
          - name: v1.26
            version: v1.26.6
          - name: v1.27
            version: v1.27.3
          - name: v1.28
            version: v1.28.0
        tests:
          - rbac
    needs: prepare-images
    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
      - name: Create kind cluster
        shell: bash
        run: |
          set -e
          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        shell: bash
        run: |
          set -e
          make kind-load-image-archive
      - name: Install kyverno
        shell: bash
        run: |
          set -e
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Test with kuttl
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -e
          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }}  --config ./test/conformance/kuttl/_config/common.yaml
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs

  # runs conformance test suites with configuration:
  policy-library:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: policy-library
            values:
              - standard
        k8s-version:
          - name: v1.25
            version: v1.25.11
          - name: v1.26
            version: v1.26.6
          - name: v1.27
            version: v1.27.3
          - name: v1.28
            version: v1.28.0
        tests:
          - argo
          - aws
          - best-practices
          - castai
          - cert-manager
          - consul
          - external-secret-operator
          - flux
          - istio
          - karpenter
          - kasten
          - kubecost
          - kubeops
          - kubevirt
          - linkerd
          - nginx-ingress
          - openshift
          # - other
          - other/a
          - other/b-d
          - other/e-l
          - other/m-q
          - other/rec-req
          - other/res
          - other/s-z
          # - pod-security
          # - psa
          - psp-migration
          # - tekton
          # - traefik
          # - velero
    needs:
      - prepare-images
      - prepare-cli
    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
    steps:
      - name: Checkout kyverno/kyverno
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Checkout kyverno/policies
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          repository: kyverno/policies
          path: policies
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
      - name: Create kind cluster
        shell: bash
        run: |
          set -e
          export KIND_IMAGE=kindest/node:${{ matrix.k8s-version.version }}
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Download kyverno CLI archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kubectl-kyverno
      - name: Install Kyverno CLI
        shell: bash
        run: |
          set -e
          chmod +x kubectl-kyverno && mv kubectl-kyverno ./cmd/cli/kubectl-kyverno/kyverno
          echo "$PWD/cmd/cli/kubectl-kyverno" >> $GITHUB_PATH
      - name: Load kyverno images archive in kind cluster
        shell: bash
        run: |
          set -e
          make kind-load-image-archive
      - name: Install kyverno
        shell: bash
        run: |
          set -e
          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Test policy library with kuttl
        shell: bash
        run: |
          set -e
          cd policies
          ../.tools/kubectl-kuttl test ./${{ matrix.tests }} --config ./kuttl-test.yaml
      - name: Debug failure
        if: failure()
        uses: ./.github/actions/kyverno-logs

  monitor-helm-secret-size:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    needs: prepare-images
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Setup caches
        uses: ./.github/actions/setup-caches
        timeout-minutes: 5
        continue-on-error: true
        with:
          build-cache-key: run-conformance
      - name: Setup build env
        uses: ./.github/actions/setup-build-env
        timeout-minutes: 10
      - name: Create kind cluster
        shell: bash
        run: |
          set -e
          make kind-create-cluster
      - name: Download kyverno images archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kyverno.tar
      - name: Load kyverno images archive in kind cluster
        shell: bash
        run: |
          set -e
          make kind-load-image-archive
      - name: Install kyverno
        shell: bash
        run: |
          make kind-install-kyverno
      - name: Wait for kyverno ready
        uses: ./.github/actions/kyverno-wait-ready
      - name: Check secret size
        shell: bash
        run: |
          set -e
          set -u
          SIZE=$(kubectl get secrets -n kyverno sh.helm.release.v1.kyverno.v1 -o jsonpath='{.data.release}' | base64 -d | wc -c | awk '{print $1}')
          MAX_ALLOWED=1030000
          if [ "$SIZE" -gt "$MAX_ALLOWED" ]; then
            echo "Helm secret size ($SIZE bytes) is above the max allowed ($MAX_ALLOWED bytes)"
            exit 1
          else
            echo "Helm secret size ($SIZE bytes) is below the max allowed ($MAX_ALLOWED bytes)"
          fi

  check-tests:
    runs-on: ubuntu-latest
    permissions:
      packages: read
    needs: prepare-cli
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - name: Download kyverno CLI archive
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
        with:
          name: kubectl-kyverno
      - name: Install Kyverno CLI
        shell: bash
        run: |
          set -e
          chmod +x kubectl-kyverno && mv kubectl-kyverno ./cmd/cli/kubectl-kyverno/kyverno
          echo "$PWD/cmd/cli/kubectl-kyverno" >> $GITHUB_PATH
      - name: Fix test files
        shell: bash
        run: |
          set -e
          KYVERNO_EXPERIMENTAL=true kyverno fix test ./test/cli --save --compress
          make verify-cli-tests

  required:
    needs:
    - standard
    - ttl
    - force-failure-policy-ignore
    - validating-admission-policies-v1alpha1
    - validating-admission-policies-v1beta1
    - validating-admission-policy-reports-v1alpha1
    - validating-admission-policy-reports-v1beta1
    - custom-sigstore
    - default
    - monitor-helm-secret-size
    - check-tests
    runs-on: ubuntu-latest
    steps:
    - run: echo "Required jobs success!"