merge upstream master

.dockerignore

@@ -1 +1,31 @@
-vendor
+.github/
+.idea/
+.vscode/
+bin/
+charts/
+docs/
+config/
+examples/
+hack/
+manifest/
+spark-docker/
+sparkctl/
+test/
+vendor/
+.dockerignore
+.DS_Store
+.gitignore
+.gitlab-ci.yaml
+.golangci.yaml
+.pre-commit-config.yaml
+ADOPTERS.md
+CODE_OF_CONDUCT.md
+codecov.ymal
+CONTRIBUTING.md
+cover.out
+Dockerfile
+LICENSE
+OWNERS
+PROJECT
+README.md
+test.sh

.github/workflows/main.yaml → .github/workflows/integration.yaml

@@ -1,22 +1,31 @@
-name: Pre-commit checks
+name: Integration Test
 
 on:
   pull_request:
     branches:
       - master
+      - release-*
 
   push:
     branches:
       - master
+      - release-*
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.actor }}
+  cancel-in-progress: true
 
 jobs:
   build-api-docs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
         with:
-          fetch-depth: "0"
+          go-version-file: go.mod
 
       - name: The API documentation hasn't changed
         run: |
@@ -32,43 +41,39 @@ jobs:
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
-        with:
-          fetch-depth: "0"
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: "go.mod"
+          go-version-file: go.mod
 
       - name: build sparkctl
-        run: |
-          make all
+        run: make build-sparkctl
 
   build-spark-operator:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
         with:
-          fetch-depth: "0"
+          fetch-depth: 0
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: "go.mod"
+          go-version-file: go.mod
 
-      - name: Run gofmt check
-        run: make fmt-check
+      - name: Run go fmt check
+        run: make go-fmt
 
-      - name: Run static analysis
-        run: make static-analysis
+      - name: Run go vet check
+        run: make go-vet
 
       - name: Run unit tests
         run: make unit-test
 
       - name: Build Spark-Operator Docker Image
-        run: |
-          docker build -t docker.io/kubeflow/spark-operator:latest .
+        run: make docker-build IMAGE_TAG=latest
 
       - name: Check changes in resources used in docker file
         run: |
@@ -88,53 +93,71 @@ jobs:
   build-helm-chart:
     runs-on: ubuntu-20.04
     steps:
+      - name: Determine branch name
+        id: get_branch
+        run: |
+          BRANCH=""
+          if [ "${{ github.event_name }}" == "push" ]; then
+            BRANCH=${{ github.ref_name }}
+          elif [ "${{ github.event_name }}" == "pull_request" ]; then
+            BRANCH=${{ github.base_ref }}
+          fi
+          echo "Branch name: $BRANCH"
+          echo "BRANCH=$BRANCH" >> "$GITHUB_OUTPUT"
+
       - name: Checkout source code
         uses: actions/checkout@v4
         with:
-          fetch-depth: "0"
+          fetch-depth: 0
 
       - name: Install Helm
         uses: azure/setup-helm@v4
         with:
           version: v3.14.3
 
-      - name: Produce the helm documentation
-        run: |
-          make helm-docs
-          if ! git diff --quiet -- charts/spark-operator-chart/README.md; then
-            echo "Need to re-run 'make helm-docs' and commit the changes"
-            false
-          fi
-
       - name: Set up chart-testing
         uses: helm/[email protected]
 
-      - name: Print chart-testing version information
-        run: ct version
-
-      - name: Run chart-testing (lint)
-        run: ct lint
-
       - name: Run chart-testing (list-changed)
         id: list-changed
+        env:
+          BRANCH: ${{ steps.get_branch.outputs.BRANCH }}
         run: |
-          changed=$(ct list-changed)
+          changed=$(ct list-changed --target-branch $BRANCH)
           if [[ -n "$changed" ]]; then
-            echo "::set-output name=changed::true"
+            echo "changed=true" >> "$GITHUB_OUTPUT"
           fi
 
+      - name: Run chart-testing (lint)
+        if: steps.list-changed.outputs.changed == 'true'
+        env:
+            BRANCH: ${{ steps.get_branch.outputs.BRANCH }}
+        run: ct lint --check-version-increment=false --target-branch $BRANCH
+
       - name: Detect CRDs drift between chart and manifest
+        if: steps.list-changed.outputs.changed == 'true'
         run: make detect-crds-drift
 
+      - name: Produce the helm documentation
+        if: steps.list-changed.outputs.changed == 'true'
+        run: |
+          make helm-docs
+          if ! git diff --quiet -- charts/spark-operator-chart/README.md; then
+            echo "Need to re-run 'make helm-docs' and commit the changes"
+            false
+          fi
+
       - name: setup minikube
-        uses: manusa/[email protected]
+        if: steps.list-changed.outputs.changed == 'true'
+        uses: manusa/[email protected]
         with:
-          minikube version: v1.32.0
-          kubernetes version: v1.28.8
+          minikube version: v1.33.0
+          kubernetes version: v1.30.0
           start args: --memory 6g --cpus=2 --addons ingress
           github token: ${{ inputs.github-token }}
 
       - name: Run chart-testing (install)
+        if: steps.list-changed.outputs.changed == 'true'
         run: |
           docker build -t docker.io/kubeflow/spark-operator:local .
           minikube image load docker.io/kubeflow/spark-operator:local
@@ -154,10 +177,10 @@ jobs:
           go-version-file: "go.mod"
 
       - name: setup minikube
-        uses: manusa/actions-setup-minikube@v2.10.0
+        uses: manusa/actions-setup-minikube@v2.11.0
         with:
-          minikube version: v1.32.0
-          kubernetes version: v1.28.8
+          minikube version: v1.33.0
+          kubernetes version: v1.30.0
           start args: --memory 6g --cpus=2 --addons ingress
           github token: ${{ inputs.github-token }}
 

.github/workflows/push-tag.yaml

@@ -0,0 +1,44 @@
+name: Push Tag on VERSION change
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+    paths:
+      - VERSION
+
+jobs:
+  push_tag:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "[email protected]"
+
+      - name: Read version from VERSION file
+        run: |
+          VERSION=$(cat VERSION)
+          echo "VERSION=$VERSION" >> $GITHUB_ENV
+
+      - name: Check if tag exists
+        run: |
+          git fetch --tags
+          if git tag -l | grep -q "^${VERSION}$"; then
+            echo "TAG_EXISTS=true" >> $GITHUB_ENV
+          else
+            echo "TAG_EXISTS=false" >> $GITHUB_ENV
+          fi
+
+      - name: Create and push tag
+        if: env.TAG_EXISTS == 'false'
+        run: |
+          git tag -a "$VERSION" -m "Release $VERSION"
+          git push origin "$VERSION"

.github/workflows/release-charts.yaml

@@ -0,0 +1,58 @@
+name: Release Helm charts
+
+on:
+  release:
+    types: [published]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "[email protected]"
+
+      - name: Set up Helm
+        uses: azure/[email protected]
+        with:
+          version: v3.14.4
+
+      - name: Package Helm charts
+        run: |
+          for chart in $(ls charts); do
+            helm package charts/$chart
+          done
+
+      - name: Save packaged charts to temp directory
+        run: |
+          mkdir -p /tmp/charts
+          cp *.tgz /tmp/charts
+
+      - name: Checkout to branch gh-pages
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+          fetch-depth: 0
+
+      - name: Copy packages charts
+        run: |
+          cp /tmp/charts/*.tgz .
+
+      - name: Update Helm charts repo index
+        env:
+          CHART_URL: https://github.com/${{ github.repository }}/releases/download/${{ github.ref_name }}
+        run: |
+          helm repo index --merge index.yaml --url $CHART_URL .
+          git add index.yaml
+          git commit -s -m "Update index.yaml" || exit 0
+          git push