kfp-tekton-unittests.yml

name: KFP Tekton Unit Tests

on:
  push:
    branches: [master]

  # Run tests for any PRs.
  pull_request:

env:
  GITHUB_ACTION: "true"
  SETUPTOOLS_USE_DISTUTILS: "stdlib"

jobs:
  python-unittest:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        env:
          SETUPTOOLS_USE_DISTUTILS: ""
        run: python -m pip install -e sdk/python
      - name: Unit Tests
        run: VENV=$VIRTUAL_ENV make ci_unit_test
  validate-testdata:
    runs-on: ubuntu-latest
    steps:
      - name: Install Go
        uses: actions/setup-go@v4
        with:
          go-version: 1.19.x
      - uses: actions/checkout@v4
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install dependencies
        run: python -m pip install -e sdk/python
      - name: "Generate testdata yaml files."
        run: VENV=$VIRTUAL_ENV make unit_test GENERATE_GOLDEN_YAML=True
      - name: "Test only required noninled yaml files are generated."
        run: make validate-generated-test-yamls
      - name: "Tekton validation for testdata."
        run: make validate-testdata
      - name: "Validation for examples data."
        run: make validate-pipelineloop-examples
  progress-report:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python 3.8
      uses: actions/setup-python@v4
      with:
        python-version: 3.8
    - name: Install dependencies
      run: python -m pip install -e sdk/python
    - name: "Progress report on compiling KFP DSL test scripts"
      run: VENV=$VIRTUAL_ENV make report
  python-lint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python 3.8
      uses: actions/setup-python@v4
      with:
        python-version: 3.8
    - name: "Lint Python code with flake8"
      run: VENV=$VIRTUAL_ENV make lint
  check-license:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    - name: "Verify source files contain the license header"
      run: make check_license
  check-mdtoc:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    - name: "Verify Markdown files have current table of contents"
      run: make check_mdtoc
  check-doc-links:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: "Verify Markdown files have valid links"
      run: make check_doc_links
  run-go-unittests:
    runs-on: ubuntu-latest
    steps:
    - name: Install Go
      uses: actions/setup-go@v4
      with:
        go-version: 1.19.x
    - name: Checkout code
      uses: actions/checkout@v4
    - name: "run go unit tests"
      run: make run-go-unittests
    - name: "Verify apiserver, agent, and workflow build"
      run: make build-backend
  run-pipelineloop-unittests:
    runs-on: ubuntu-latest
    steps:
      - name: Install Go
        uses: actions/setup-go@v4
        with:
          go-version: 1.19.x
      - name: Checkout code
        uses: actions/checkout@v4
      - name: "run go pipelineLoop unit tests"
        run: cd tekton-catalog/pipeline-loops && make test-all
  run-v2-custom-controller-image-builds:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - id: changes
        uses: jitterbit/get-changed-files@v1
        with:
          format: 'json'
      - name: backend changes
        id: backend-changes
        run: |
          readarray -t backend_files <<<"$(jq -r '.[]|select(startswith("tekton-catalog"))|select(endswith("README.md")|not)' <<<'${{ steps.changes.outputs.all }}')"
          echo "::set-output name=all::"
          if [[ ${#backend_files[@]} -gt 0 ]]; then
            if [[ -n "${backend_files[0]}" ]]; then
              echo "::set-output name=all::yes"
            fi
          fi
      - name: build images
        if: ${{ steps.backend-changes.outputs.all }}
        run: make build-v2-custom-controller-images
  backend-integration:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - id: changes
        uses: jitterbit/get-changed-files@v1
        with:
          format: 'json'
      - name: backend changes
        id: backend-changes
        run: |
          readarray -t backend_files <<<"$(jq -r '.[]|select(startswith("backend") or startswith("go.mod"))|select(endswith("README.md")|not)' <<<'${{ steps.changes.outputs.all }}')"
          echo "::set-output name=all::"
          if [[ ${#backend_files[@]} -gt 0 ]]; then
            if [[ -n "${backend_files[0]}" ]]; then
              echo "::set-output name=all::yes"
            fi
          fi
      - name: Create k8s Kind Cluster
        if: ${{ steps.backend-changes.outputs.all }}
        uses: container-tools/kind-action@v2
        with:
          cluster_name: kfp-tekton
          kubectl_version: v1.26.1
          version: v0.17.0
          node_image: kindest/node:v1.26.0
      - name: build images
        if: ${{ steps.backend-changes.outputs.all }}
        run: ./scripts/deploy/github/build-images.sh
      - name: Set up Python 3.9
        if: ${{ steps.backend-changes.outputs.all }}
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - name: "deploy kfp-tekton"
        if: ${{ steps.backend-changes.outputs.all }}
        run: ./scripts/deploy/github/deploy-kfp.sh
      - name: Install sdk
        if: ${{ steps.backend-changes.outputs.all }}
        run: python -m pip install -e sdk/python
      - name: "flip coin test"
        if: ${{ steps.backend-changes.outputs.all }}
        run: ./scripts/deploy/github/e2e-test.sh