Merge pull request #93 from ty-dc/ci/upgrade_test

Add upgrade test for spiderpool

.github/workflows/upgrade-ci.yaml

@@ -0,0 +1,308 @@
+name: Upgrade test
+
+permissions: write-all
+
+env:
+  CLUSTER_NAME: spider
+  E2E_TIME_OUT: 60m
+
+on:
+  schedule:
+    - cron: "0 20 * * *"
+
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'sha, tag, branch'
+        required: true
+        default: main
+      spider_version:
+        description: 'It should be the released version, for example: v0.7.3; if not set, the default version set will be run.'
+        required: false
+        type: string
+        default: v0.7.3
+      e2e_enabled:
+        description: 'run e2e test'
+        required: false
+        type: choice
+        default: "false"
+        options:
+          - "true"
+          - "false"
+
+jobs:
+  get_ref:
+    runs-on: ubuntu-latest
+    outputs:
+      ref: ${{ env.RUN_REF }}
+      e2e_enabled: ${{ env.RUN_E2E_ENABLED }}
+      default_spiderpool_version: ${{ env.DEFAULT_SPIDERPOOL_VERSION }}
+    steps:
+      - name: Get Ref
+        id: get_ref
+        run: |
+          if ${{ github.event_name == 'workflow_dispatch' }} ; then
+            echo "call by self workflow_dispatch"
+            echo "RUN_TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV
+            if ${{ github.event.inputs.spider_version == '' }}; then
+              echo "DEFAULT_SPIDERPOOL_VERSION=true" >> $GITHUB_ENV
+            else
+              echo "A custom version of k8s will be run: ${{ github.event.inputs.spider_version }} "
+              echo "DEFAULT_SPIDERPOOL_VERSION=false" >> $GITHUB_ENV
+            fi
+            if ${{ github.event.inputs.e2e_enabled == 'true' }}; then
+              echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
+            else
+              echo "RUN_E2E_ENABLED=false" >> $GITHUB_ENV
+            fi
+          else
+            # schedule event
+            # use main sha for ci image tag
+            echo "trigger by schedule"
+            echo "RUN_TAG=main" >> $GITHUB_ENV
+            echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
+            echo "DEFAULT_SPIDERPOOL_VERSION=true" >> $GITHUB_ENV
+          fi
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ env.RUN_TAG }}
+
+      - name: Result Ref
+        id: result
+        run: |
+          ref=$( git show -s --format='format:%H')
+          echo "RUN_REF=${ref}" >> $GITHUB_ENV
+
+  call_build_ci_image:
+    needs: [get_ref]
+    # get  image:${{ needs.get_ref.outputs.ref }} and image-ci:${{ needs.get_ref.outputs.ref }}
+    uses: ./.github/workflows/build-image-ci.yaml
+    with:
+      ref: ${{ needs.get_ref.outputs.ref }}
+      push: false
+    secrets: inherit
+
+  lint_chart_against_release_image:
+    needs: get_ref
+    uses: ./.github/workflows/call-lint-chart.yaml
+    with:
+      ref: ${{ needs.get_ref.outputs.ref }}
+    secrets: inherit
+
+  call_release_chart:
+    needs: [get_ref]
+    uses: ./.github/workflows/call-release-chart.yaml
+    with:
+      ref: ${{ needs.get_ref.outputs.ref }}
+      submit: false
+    secrets: inherit
+
+  trivy_scan_images:
+    needs: [call_build_ci_image, get_ref, call_release_chart]
+    uses: ./.github/workflows/trivy-scan-image.yaml
+    with:
+      image_tag: ${{ needs.call_build_ci_image.outputs.imageTag }}
+      ref: ${{ needs.get_ref.outputs.ref }}
+    secrets: inherit
+
+  run_upgrade_tests_on_release_version:
+    needs: [call_build_ci_image, get_ref, call_release_chart]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # Synchronise with the latest releases of each version
+        version: [release-v0.7, release-v0.8, release-v0.9]
+        include:
+          - e2e_init_mode: e2e_init_underlay
+            e2e_test_mode: e2e_test_underlay
+          - e2e_init_mode: e2e_init_cilium_with_ebpf
+            e2e_test_mode: e2e_test_cilium_with_ebpf
+          - e2e_init_mode: e2e_init_calico
+            e2e_test_mode: e2e_test_calico
+          - e2e_init_mode: e2e_init_cilium
+            e2e_test_mode: e2e_test_cilium
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ matrix.version }}
+
+      - name: Get image tag
+        run: |
+          cat VERSION
+          imageTag=$(grep -E "^[[:space:]]*v[0-9]+.[0-9]+.[0-9]+[[:space:]]*$" VERSION)
+          echo "imageTag=$imageTag" >> $GITHUB_ENV
+
+      - name: Prepare
+        id: prepare
+        run: |
+          echo "ref: ${{ inputs.ref }} "
+          echo "===== image "
+          echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller-ci:${{ env.imageTag }}"
+          echo "ci race image tag: ghcr.io/${{ github.repository }}/spiderpool-controller-ci:${{ env.imageTag }}-race"
+          echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-agent-ci:${{ env.imageTag }}"
+          echo "ci race image tag: ghcr.io/${{ github.repository }}/spiderpool-agent-ci:${{ env.imageTag }}-race"
+          TMP=` date +%m%d%H%M%S `
+          E2E_CLUSTER_NAME="spiderpool${TMP}"
+          echo "E2E_CLUSTER_NAME=${E2E_CLUSTER_NAME}" >> $GITHUB_ENV
+
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21.4
+
+      # https://github.com/helm/kind-action
+      - name: Install Kind Bin
+        uses: helm/[email protected]
+        with:
+          install_only: true
+
+      - name: Install Tools
+        run: |
+          # install kind/p2ctl/helm/ginkgo
+          bash ./test/scripts/install-tools.sh
+
+      # test against commit version
+      # https://github.com/kubernetes-sigs/kind/issues/2863
+      - name: Setup Kind Cluster on ${{ matrix.version }}
+        uses: nick-invision/retry@v2
+        with:
+          timeout_minutes: 20
+          max_attempts: 3
+          command: |
+            make ${{ matrix.e2e_init_mode }} -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \
+                -e E2E_SPIDERPOOL_TAG=${{ env.imageTag }} \
+                -e PYROSCOPE_LOCAL_PORT="" \
+                -e INSTALL_KUBEVIRT=true \
+                -e INSTALL_KRUISE=true \
+                -e INSTALL_KDOCTOR=true \
+                -e INSTALL_OVS=true \
+                -e INSTALL_RDMA=true \
+                -e INSTALL_SRIOV=true
+
+      - name: Run e2e Test on ${{ matrix.version }}
+        id: pre_run_e2e
+        continue-on-error: true
+        if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }}
+        run: |
+          E2E_LABELS=${{ inputs.e2e_labels }}
+          echo "run e2e labels: ${E2E_LABELS}"
+          RESULT=0
+          make ${{ matrix.e2e_test_mode }} -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }}  \
+              -e E2E_GINKGO_LABELS=${E2E_LABELS} \
+              -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }} \
+              -e E2E_IP_FAMILY=${{ inputs.ip_family }} || RESULT=1
+          if ((RESULT==0)) ; then
+              echo "RUN_E2E_PASS=true" >> $GITHUB_ENV
+          else
+              echo "RUN_E2E_PASS=false" >> $GITHUB_ENV
+          fi
+          if [ -f "test/e2edebugLog" ] ; then
+              echo "UPLOAD_E2E_LOG=true" >> $GITHUB_ENV
+          else
+              echo "UPLOAD_E2E_LOG=false" >> $GITHUB_ENV
+          fi
+          if [ -f "./e2ereport.json" ] ; then
+              echo "error, did not find e2e report"
+              echo "UPLOAD_E2E_REPORT=true" >> $GITHUB_ENV
+          else
+              echo "UPLOAD_E2E_REPORT=false" >> $GITHUB_ENV
+          fi
+
+      - name: Upload e2e log
+        if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }}
+        uses: actions/[email protected]
+        with:
+          name: ${{ matrix.version }}-${{ matrix.e2e_test_mode }}-debuglog.txt
+          path: test/e2edebugLog.txt
+          retention-days: 7
+
+      - name: Upload e2e report
+        if: ${{ env.UPLOAD_E2E_REPORT == 'true' }}
+        uses: actions/[email protected]
+        with:
+          name: ${{ matrix.version }}-${{ matrix.e2e_test_mode }}-debuglog.txt-e2ereport.json
+          path: e2ereport.json
+          retention-days: 1
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ env.RUN_REF }}
+
+      - name: Upgrade to version ${{ env.RUN_REF }}
+        continue-on-error: true
+        run: |
+          make upgrade_e2e_spiderpool
+
+      - name: Run e2e Test on ${{ env.RUN_REF }}
+        id: run_e2e
+        continue-on-error: true
+        if: ${{ always() && steps.pre_run_e2e.result != 'failure' && needs.get_ref.outputs.e2e_enabled == 'true' }}
+        run: |
+          E2E_LABELS=${{ inputs.e2e_labels }}
+          echo "run e2e labels: ${E2E_LABELS}"
+          RESULT=0
+          make ${{ matrix.e2e_test_mode }} -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }}  \
+              -e E2E_GINKGO_LABELS=${E2E_LABELS} \
+              -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }} \
+              -e E2E_IP_FAMILY=${{ inputs.ip_family }} || RESULT=1
+          if ((RESULT==0)) ; then
+              echo "RUN_E2E_PASS=true" >> $GITHUB_ENV
+          else
+              echo "RUN_E2E_PASS=false" >> $GITHUB_ENV
+          fi
+          if [ -f "test/e2edebugLog" ] ; then
+              echo "UPLOAD_E2E_LOG=true" >> $GITHUB_ENV
+          else
+              echo "UPLOAD_E2E_LOG=false" >> $GITHUB_ENV
+          fi
+          if [ -f "./e2ereport.json" ] ; then
+              echo "error, did not find e2e report"
+              echo "UPLOAD_E2E_REPORT=true" >> $GITHUB_ENV
+          else
+              echo "UPLOAD_E2E_REPORT=false" >> $GITHUB_ENV
+          fi
+
+      - name: Upload e2e log
+        if: ${{ always() && steps.pre_run_e2e.result != 'failure' && needs.get_ref.outputs.e2e_enabled == 'true' }}
+        uses: actions/[email protected]
+        with:
+          name: ${{ env.RUN_REF }}-${{ matrix.e2e_test_mode }}-debuglog.txt
+          path: test/e2edebugLog.txt
+          retention-days: 7
+
+      - name: Upload e2e report
+        if: ${{ env.UPLOAD_E2E_REPORT == 'true' }}
+        uses: actions/[email protected]
+        with:
+          name: ${{ env.RUN_REF }}-${{ matrix.e2e_test_mode }}-debuglog.txt-e2ereport.json
+          path: e2ereport.json
+          retention-days: 1
+
+  creat_issue:
+    runs-on: ubuntu-latest
+    needs: [run_upgrade_tests_on_release_version]
+    if: ${{ always() && run_upgrade_tests_on_release_version == 'failure' }}
+    steps:
+      - name: echo
+        run: |
+          echo ${{ github.repository }}
+          echo ${{ github.repository_owner }}
+          echo "TIMESTAMP=`date +%Y-%m-%d`" >> $GITHUB_ENV
+
+      - name: create an issue
+        uses: dacbd/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title: "Nightly upgrade test CI ${{ ENV.TIMESTAMP }}: Failed"
+          body: |
+            action url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          labels: "kind/ci-bug"
+          assignees: "Icarus9913,ty-dc"

Makefile

@@ -428,6 +428,10 @@ clean: clean_e2e
 clean_e2e_spiderpool:
 	-$(QUIET) make -C test uninstall_spiderpool
 
+.PHONY: upgrade_e2e_spiderpool
+upgrade_e2e_spiderpool:
+	-$(QUIET) make -C test upgrade_spiderpool
+
 .PHONY: codegen
 codegen:
 	@echo "Generate k8s SDK with code-generator."

test/Makefile

@@ -389,6 +389,20 @@ uninstall_spiderpool:
 	@ make helm_uninstall_spiderpool
 	@ make clean_spiderpool_crd
 
+.PHONY: upgrade_spiderpool
+upgrade_spiderpool:
+	@echo -e "\033[35m [upgrade spiderpool] \033[0m"
+	@make helm_upgrade_spiderpool
+
+.PHONY: helm_upgrade_spiderpool
+helm_upgrade_spiderpool:
+	@echo -e "\033[35m [helm upgrade spiderpool] \033[0m"
+	cd $(ROOT_DIR)/charts/spiderpool/crds ; \
+	ls | grep '\.yaml$$' | xargs -n1 -P0 -i kubectl --kubeconfig $(E2E_KUBECONFIG) apply -f {} ; \
+	kubectl delete po -n $(RELEASE_NAMESPACE) spiderpool-init --kubeconfig $(E2E_KUBECONFIG) || true
+	helm --kubeconfig $(E2E_KUBECONFIG) upgrade  $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool \
+		-n $(RELEASE_NAMESPACE) --reuse-values || { KIND_CLUSTER_NAME=$(E2E_CLUSTER_NAME) ./scripts/debugEnv.sh $(E2E_KUBECONFIG) "detail"; exit 1; }
+
 .PHONY: clean
 clean:
 	@rm -rf $(CLUSTER_DIR)