From 124c2e2a6badc0083e8c4d9104a1dffab33b4aa5 Mon Sep 17 00:00:00 2001 From: ty-dc Date: Thu, 14 Mar 2024 10:47:52 +0800 Subject: [PATCH] Add upgrade test for spiderpool Signed-off-by: ty-dc --- .github/workflows/upgrade-ci.yaml | 308 ++++++++++++++++++++++++++++++ Makefile | 4 + test/Makefile | 14 ++ 3 files changed, 326 insertions(+) create mode 100644 .github/workflows/upgrade-ci.yaml diff --git a/.github/workflows/upgrade-ci.yaml b/.github/workflows/upgrade-ci.yaml new file mode 100644 index 0000000000..4204484046 --- /dev/null +++ b/.github/workflows/upgrade-ci.yaml @@ -0,0 +1,308 @@ +name: Upgrade test + +permissions: write-all + +env: + CLUSTER_NAME: spider + E2E_TIME_OUT: 60m + +on: + schedule: + - cron: "0 20 * * *" + + workflow_dispatch: + inputs: + ref: + description: 'sha, tag, branch' + required: true + default: main + spider_version: + description: 'It should be the released version, for example: v0.7.3; if not set, the default version set will be run.' + required: false + type: string + default: v0.7.3 + e2e_enabled: + description: 'run e2e test' + required: false + type: choice + default: "false" + options: + - "true" + - "false" + +jobs: + get_ref: + runs-on: ubuntu-latest + outputs: + ref: ${{ env.RUN_REF }} + e2e_enabled: ${{ env.RUN_E2E_ENABLED }} + default_spiderpool_version: ${{ env.DEFAULT_SPIDERPOOL_VERSION }} + steps: + - name: Get Ref + id: get_ref + run: | + if ${{ github.event_name == 'workflow_dispatch' }} ; then + echo "call by self workflow_dispatch" + echo "RUN_TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV + if ${{ github.event.inputs.spider_version == '' }}; then + echo "DEFAULT_SPIDERPOOL_VERSION=true" >> $GITHUB_ENV + else + echo "A custom version of k8s will be run: ${{ github.event.inputs.spider_version }} " + echo "DEFAULT_SPIDERPOOL_VERSION=false" >> $GITHUB_ENV + fi + if ${{ github.event.inputs.e2e_enabled == 'true' }}; then + echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV + else + echo "RUN_E2E_ENABLED=false" >> $GITHUB_ENV + fi + else + # schedule event + # use main sha for ci image tag + echo "trigger by schedule" + echo "RUN_TAG=main" >> $GITHUB_ENV + echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV + echo "DEFAULT_SPIDERPOOL_VERSION=true" >> $GITHUB_ENV + fi + + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ env.RUN_TAG }} + + - name: Result Ref + id: result + run: | + ref=$( git show -s --format='format:%H') + echo "RUN_REF=${ref}" >> $GITHUB_ENV + + call_build_ci_image: + needs: [get_ref] + # get image:${{ needs.get_ref.outputs.ref }} and image-ci:${{ needs.get_ref.outputs.ref }} + uses: ./.github/workflows/build-image-ci.yaml + with: + ref: ${{ needs.get_ref.outputs.ref }} + push: false + secrets: inherit + + lint_chart_against_release_image: + needs: get_ref + uses: ./.github/workflows/call-lint-chart.yaml + with: + ref: ${{ needs.get_ref.outputs.ref }} + secrets: inherit + + call_release_chart: + needs: [get_ref] + uses: ./.github/workflows/call-release-chart.yaml + with: + ref: ${{ needs.get_ref.outputs.ref }} + submit: false + secrets: inherit + + trivy_scan_images: + needs: [call_build_ci_image, get_ref, call_release_chart] + uses: ./.github/workflows/trivy-scan-image.yaml + with: + image_tag: ${{ needs.call_build_ci_image.outputs.imageTag }} + ref: ${{ needs.get_ref.outputs.ref }} + secrets: inherit + + run_upgrade_tests_on_release_version: + needs: [call_build_ci_image, get_ref, call_release_chart] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Synchronise with the latest releases of each version + version: [release-v0.7, release-v0.8, release-v0.9] + include: + - e2e_init_mode: e2e_init_underlay + e2e_test_mode: e2e_test_underlay + - e2e_init_mode: e2e_init_cilium_with_ebpf + e2e_test_mode: e2e_test_cilium_with_ebpf + - e2e_init_mode: e2e_init_calico + e2e_test_mode: e2e_test_calico + - e2e_init_mode: e2e_init_cilium + e2e_test_mode: e2e_test_cilium + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ matrix.version }} + + - name: Get image tag + run: | + cat VERSION + imageTag=$(grep -E "^[[:space:]]*v[0-9]+.[0-9]+.[0-9]+[[:space:]]*$" VERSION) + echo "imageTag=$imageTag" >> $GITHUB_ENV + + - name: Prepare + id: prepare + run: | + echo "ref: ${{ inputs.ref }} " + echo "===== image " + echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller-ci:${{ env.imageTag }}" + echo "ci race image tag: ghcr.io/${{ github.repository }}/spiderpool-controller-ci:${{ env.imageTag }}-race" + echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-agent-ci:${{ env.imageTag }}" + echo "ci race image tag: ghcr.io/${{ github.repository }}/spiderpool-agent-ci:${{ env.imageTag }}-race" + TMP=` date +%m%d%H%M%S ` + E2E_CLUSTER_NAME="spiderpool${TMP}" + echo "E2E_CLUSTER_NAME=${E2E_CLUSTER_NAME}" >> $GITHUB_ENV + + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 1.21.4 + + # https://github.com/helm/kind-action + - name: Install Kind Bin + uses: helm/kind-action@v1.8.0 + with: + install_only: true + + - name: Install Tools + run: | + # install kind/p2ctl/helm/ginkgo + bash ./test/scripts/install-tools.sh + + # test against commit version + # https://github.com/kubernetes-sigs/kind/issues/2863 + - name: Setup Kind Cluster on ${{ matrix.version }} + uses: nick-invision/retry@v2 + with: + timeout_minutes: 20 + max_attempts: 3 + command: | + make ${{ matrix.e2e_init_mode }} -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \ + -e E2E_SPIDERPOOL_TAG=${{ env.imageTag }} \ + -e PYROSCOPE_LOCAL_PORT="" \ + -e INSTALL_KUBEVIRT=true \ + -e INSTALL_KRUISE=true \ + -e INSTALL_KDOCTOR=true \ + -e INSTALL_OVS=true \ + -e INSTALL_RDMA=true \ + -e INSTALL_SRIOV=true + + - name: Run e2e Test on ${{ matrix.version }} + id: pre_run_e2e + continue-on-error: true + if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} + run: | + E2E_LABELS=${{ inputs.e2e_labels }} + echo "run e2e labels: ${E2E_LABELS}" + RESULT=0 + make ${{ matrix.e2e_test_mode }} -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \ + -e E2E_GINKGO_LABELS=${E2E_LABELS} \ + -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }} \ + -e E2E_IP_FAMILY=${{ inputs.ip_family }} || RESULT=1 + if ((RESULT==0)) ; then + echo "RUN_E2E_PASS=true" >> $GITHUB_ENV + else + echo "RUN_E2E_PASS=false" >> $GITHUB_ENV + fi + if [ -f "test/e2edebugLog" ] ; then + echo "UPLOAD_E2E_LOG=true" >> $GITHUB_ENV + else + echo "UPLOAD_E2E_LOG=false" >> $GITHUB_ENV + fi + if [ -f "./e2ereport.json" ] ; then + echo "error, did not find e2e report" + echo "UPLOAD_E2E_REPORT=true" >> $GITHUB_ENV + else + echo "UPLOAD_E2E_REPORT=false" >> $GITHUB_ENV + fi + + - name: Upload e2e log + if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} + uses: actions/upload-artifact@v3.1.3 + with: + name: ${{ matrix.version }}-${{ matrix.e2e_test_mode }}-debuglog.txt + path: test/e2edebugLog.txt + retention-days: 7 + + - name: Upload e2e report + if: ${{ env.UPLOAD_E2E_REPORT == 'true' }} + uses: actions/upload-artifact@v3.1.3 + with: + name: ${{ matrix.version }}-${{ matrix.e2e_test_mode }}-debuglog.txt-e2ereport.json + path: e2ereport.json + retention-days: 1 + + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ env.RUN_REF }} + + - name: Upgrade to version ${{ env.RUN_REF }} + continue-on-error: true + run: | + make upgrade_e2e_spiderpool + + - name: Run e2e Test on ${{ env.RUN_REF }} + id: run_e2e + continue-on-error: true + if: ${{ always() && steps.pre_run_e2e.result != 'failure' && needs.get_ref.outputs.e2e_enabled == 'true' }} + run: | + E2E_LABELS=${{ inputs.e2e_labels }} + echo "run e2e labels: ${E2E_LABELS}" + RESULT=0 + make ${{ matrix.e2e_test_mode }} -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \ + -e E2E_GINKGO_LABELS=${E2E_LABELS} \ + -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }} \ + -e E2E_IP_FAMILY=${{ inputs.ip_family }} || RESULT=1 + if ((RESULT==0)) ; then + echo "RUN_E2E_PASS=true" >> $GITHUB_ENV + else + echo "RUN_E2E_PASS=false" >> $GITHUB_ENV + fi + if [ -f "test/e2edebugLog" ] ; then + echo "UPLOAD_E2E_LOG=true" >> $GITHUB_ENV + else + echo "UPLOAD_E2E_LOG=false" >> $GITHUB_ENV + fi + if [ -f "./e2ereport.json" ] ; then + echo "error, did not find e2e report" + echo "UPLOAD_E2E_REPORT=true" >> $GITHUB_ENV + else + echo "UPLOAD_E2E_REPORT=false" >> $GITHUB_ENV + fi + + - name: Upload e2e log + if: ${{ always() && steps.pre_run_e2e.result != 'failure' && needs.get_ref.outputs.e2e_enabled == 'true' }} + uses: actions/upload-artifact@v3.1.3 + with: + name: ${{ env.RUN_REF }}-${{ matrix.e2e_test_mode }}-debuglog.txt + path: test/e2edebugLog.txt + retention-days: 7 + + - name: Upload e2e report + if: ${{ env.UPLOAD_E2E_REPORT == 'true' }} + uses: actions/upload-artifact@v3.1.3 + with: + name: ${{ env.RUN_REF }}-${{ matrix.e2e_test_mode }}-debuglog.txt-e2ereport.json + path: e2ereport.json + retention-days: 1 + + creat_issue: + runs-on: ubuntu-latest + needs: [run_upgrade_tests_on_release_version] + if: ${{ always() && run_upgrade_tests_on_release_version == 'failure' }} + steps: + - name: echo + run: | + echo ${{ github.repository }} + echo ${{ github.repository_owner }} + echo "TIMESTAMP=`date +%Y-%m-%d`" >> $GITHUB_ENV + + - name: create an issue + uses: dacbd/create-issue-action@v1.2.1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + title: "Nightly upgrade test CI ${{ ENV.TIMESTAMP }}: Failed" + body: | + action url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + labels: "kind/ci-bug" + assignees: "Icarus9913,ty-dc" diff --git a/Makefile b/Makefile index 1cc324880e..d22ab56bd7 100644 --- a/Makefile +++ b/Makefile @@ -428,6 +428,10 @@ clean: clean_e2e clean_e2e_spiderpool: -$(QUIET) make -C test uninstall_spiderpool +.PHONY: upgrade_e2e_spiderpool +upgrade_e2e_spiderpool: + -$(QUIET) make -C test upgrade_spiderpool + .PHONY: codegen codegen: @echo "Generate k8s SDK with code-generator." diff --git a/test/Makefile b/test/Makefile index 68c9e45df3..a0537a6927 100644 --- a/test/Makefile +++ b/test/Makefile @@ -389,6 +389,20 @@ uninstall_spiderpool: @ make helm_uninstall_spiderpool @ make clean_spiderpool_crd +.PHONY: upgrade_spiderpool +upgrade_spiderpool: + @echo -e "\033[35m [upgrade spiderpool] \033[0m" + @make helm_upgrade_spiderpool + +.PHONY: helm_upgrade_spiderpool +helm_upgrade_spiderpool: + @echo -e "\033[35m [helm upgrade spiderpool] \033[0m" + cd $(ROOT_DIR)/charts/spiderpool/crds ; \ + ls | grep '\.yaml$$' | xargs -n1 -P0 -i kubectl --kubeconfig $(E2E_KUBECONFIG) apply -f {} ; \ + kubectl delete po -n $(RELEASE_NAMESPACE) spiderpool-init --kubeconfig $(E2E_KUBECONFIG) || true + helm --kubeconfig $(E2E_KUBECONFIG) upgrade $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool \ + -n $(RELEASE_NAMESPACE) --reuse-values || { KIND_CLUSTER_NAME=$(E2E_CLUSTER_NAME) ./scripts/debugEnv.sh $(E2E_KUBECONFIG) "detail"; exit 1; } + .PHONY: clean clean: @rm -rf $(CLUSTER_DIR)