From 0e925b68bee4d6ea24272ce05ec878d925d048ed Mon Sep 17 00:00:00 2001 From: ty-dc Date: Thu, 14 Mar 2024 10:47:52 +0800 Subject: [PATCH] Add upgrade test for spiderpool Signed-off-by: ty-dc --- .github/workflows/auto-upgrade-ci.yaml | 264 +++++++++++++++++++++++++ Makefile | 4 + test/Makefile | 24 +++ 3 files changed, 292 insertions(+) create mode 100644 .github/workflows/auto-upgrade-ci.yaml diff --git a/.github/workflows/auto-upgrade-ci.yaml b/.github/workflows/auto-upgrade-ci.yaml new file mode 100644 index 0000000000..44e5ad13b0 --- /dev/null +++ b/.github/workflows/auto-upgrade-ci.yaml @@ -0,0 +1,264 @@ +name: Auto Upgrade Test + +permissions: write-all + +env: + CLUSTER_NAME: spider + E2E_TIME_OUT: 60m + KUBECONFIG_PATH: /home/runner/work/spiderpool/spiderpool/test/.cluster +on: + schedule: + - cron: "0 20 * * *" + + workflow_dispatch: + inputs: + ref: + description: 'sha, tag, branch. also this is the version you will upgrade to.' + required: true + default: main + e2e_enabled: + description: 'run e2e test' + required: false + type: choice + default: "false" + options: + - "true" + - "false" + +jobs: + get_ref: + runs-on: ubuntu-latest + outputs: + ref: ${{ env.RUN_REF }} + e2e_enabled: ${{ env.RUN_E2E_ENABLED }} + steps: + - name: Get Ref + id: get_ref + run: | + if ${{ github.event_name == 'workflow_dispatch' }} ; then + echo "call by self workflow_dispatch" + echo "RUN_TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV + if ${{ github.event.inputs.e2e_enabled == 'true' }}; then + echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV + else + echo "RUN_E2E_ENABLED=false" >> $GITHUB_ENV + fi + else + # schedule event + # use main sha for ci image tag + echo "trigger by schedule" + echo "RUN_TAG=main" >> $GITHUB_ENV + echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV + echo "DEFAULT_SPIDERPOOL_VERSION=true" >> $GITHUB_ENV + fi + + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ env.RUN_TAG }} + + - name: Result Ref + id: result + run: | + ref=$( git show -s --format='format:%H') + echo "RUN_REF=${ref}" >> $GITHUB_ENV + + call_build_ci_image: + needs: [get_ref] + uses: ./.github/workflows/build-image-ci.yaml + with: + ref: ${{ needs.get_ref.outputs.ref }} + push: false + secrets: inherit + + lint_chart_against_release_image: + needs: get_ref + uses: ./.github/workflows/call-lint-chart.yaml + with: + ref: ${{ needs.get_ref.outputs.ref }} + secrets: inherit + + trivy_scan_images: + needs: [call_build_ci_image, get_ref] + uses: ./.github/workflows/trivy-scan-image.yaml + with: + image_tag: ${{ needs.call_build_ci_image.outputs.imageTag }} + ref: ${{ needs.get_ref.outputs.ref }} + secrets: inherit + + run_upgrade_tests_on_release_version: + needs: [call_build_ci_image, get_ref] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Synchronise with the latest releases of each version + version: [release-v0.7, release-v0.8] + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ matrix.version }} + + - name: Get image tag + run: | + cat VERSION + imageTag=$(grep -E "^[[:space:]]*v[0-9]+.[0-9]+.[0-9]+[[:space:]]*$" VERSION) + echo "imageTag=$imageTag" >> $GITHUB_ENV + + - name: Prepare + id: prepare + run: | + echo "ref: ${{ inputs.ref }} " + echo "===== image " + echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:${{ env.imageTag }}" + TMP=` date +%m%d%H%M%S ` + E2E_CLUSTER_NAME="spiderpool${TMP}" + echo "E2E_CLUSTER_NAME=${E2E_CLUSTER_NAME}" >> $GITHUB_ENV + + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: 1.21.4 + + # https://github.com/helm/kind-action + - name: Install Kind Bin + uses: helm/kind-action@v1.8.0 + with: + install_only: true + + - name: Install Tools + run: | + # install kind/p2ctl/helm/ginkgo + bash ./test/scripts/install-tools.sh + + # test against commit version + # https://github.com/kubernetes-sigs/kind/issues/2863 + - name: Setup Kind Cluster on ${{ matrix.version }} + uses: nick-invision/retry@v2 + with: + timeout_minutes: 20 + max_attempts: 3 + command: | + make e2e_init_underlay -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \ + -e E2E_SPIDERPOOL_TAG=${{ env.imageTag }} \ + -e PYROSCOPE_LOCAL_PORT="" + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + - name: Move kubeconfig + run: | + cp -r ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config /tmp + + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ needs.get_ref.outputs.ref }} + fetch-depth: 0 + + - name: Move kubeconfig to branch ${{ needs.get_ref.outputs.ref }} + run: | + mkdir -p ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/ + cp -r /tmp/config ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config + + - name: Download spiderpool-agent image + uses: actions/download-artifact@v3 + with: + name: image-tar-spiderpool-agent + path: test/.download + + - name: Download spiderpool-controller image + uses: actions/download-artifact@v3 + with: + name: image-tar-spiderpool-controller + path: test/.download + + - name: Load Images + run: | + TAR_FILES=` ls test/.download ` + echo $TAR_FILES + for ITEM in $TAR_FILES ; do + IMAGE_NAME=${ITEM%*.tar} + echo ${IMAGE_NAME} + cat test/.download/${ITEM} | docker import - ${IMAGE_NAME}:${{ needs.call_build_ci_image.outputs.imageTag }} + done + + - name: Upgrade to version ${{ needs.get_ref.outputs.ref }} + continue-on-error: true + run: | + make upgrade_e2e_spiderpool -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \ + -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race \ + -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \ + + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ matrix.version }} + + - name: Move kubeconfig to branch ${{ needs.get_ref.outputs.ref }} + run: | + mkdir -p ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/ + cp -r /tmp/config ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config + + - name: Run e2e Test on ${{ needs.get_ref.outputs.ref }} + id: run_e2e + continue-on-error: true + if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} + run: | + RESULT=0 + export KUBECONFIG=${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config + kubectl get po -A | grep "spiderpool" + RESULT=$(kubectl get po -A | grep "spiderpool" | grep -Ev "Running|Completed" | wc -l ) + if ((RESULT==0)) ; then + echo "RUN_E2E_PASS=true" >> $GITHUB_ENV + kubectl delete po -l app=test-pod + if kubectl wait --for=condition=ready -l app=test-pod --timeout=300s pod; then + exit 0 + else + exit 1 + fi + else + echo "RUN_E2E_PASS=false" >> $GITHUB_ENV + fi + + - name: Upload e2e log + if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} + uses: actions/upload-artifact@v3.1.3 + with: + name: ${{ matrix.version }}-${{ needs.get_ref.outputs.ref }}-debuglog.txt + path: test/e2edebugLog.txt + retention-days: 7 + + - name: Upload e2e report + if: ${{ env.UPLOAD_E2E_REPORT == 'true' }} + uses: actions/upload-artifact@v3.1.3 + with: + name: ${{ matrix.version }}-${{ needs.get_ref.outputs.ref }}-e2ereport.json + path: e2ereport.json + retention-days: 1 + + - name: Show e2e Result + if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} + run: | + if ${{ env.RUN_E2E_PASS == 'true' && env.CLEAN_E2E_PASS == 'true' }} ;then + exit 0 + else + exit 1 + fi diff --git a/Makefile b/Makefile index 1cc324880e..d22ab56bd7 100644 --- a/Makefile +++ b/Makefile @@ -428,6 +428,10 @@ clean: clean_e2e clean_e2e_spiderpool: -$(QUIET) make -C test uninstall_spiderpool +.PHONY: upgrade_e2e_spiderpool +upgrade_e2e_spiderpool: + -$(QUIET) make -C test upgrade_spiderpool + .PHONY: codegen codegen: @echo "Generate k8s SDK with code-generator." diff --git a/test/Makefile b/test/Makefile index 1910e4eae5..3a238f9699 100644 --- a/test/Makefile +++ b/test/Makefile @@ -389,6 +389,30 @@ uninstall_spiderpool: @ make helm_uninstall_spiderpool @ make clean_spiderpool_crd +.PHONY: upgrade_spiderpool +upgrade_spiderpool: + @echo -e "\033[35m [upgrade spiderpool] \033[0m" + @make helm_upgrade_spiderpool + +.PHONY: helm_upgrade_spiderpool +helm_upgrade_spiderpool: + @echo -e "\033[35m [helm upgrade spiderpool] \033[0m" + cd $(ROOT_DIR)/charts/spiderpool/crds ; \ + ls | grep '\.yaml$$' | xargs -I {} kubectl apply -f {} --kubeconfig $(E2E_KUBECONFIG) ; \ + kubectl delete po -n $(RELEASE_NAMESPACE) spiderpool-init --kubeconfig $(E2E_KUBECONFIG) || true ;\ + HELM_OPTION="";\ + HELM_OPTION+=" --set spiderpoolAgent.image.registry="" \ + --set spiderpoolAgent.image.repository=$(SPIDERPOOL_AGENT_IMAGE_NAME) \ + --set spiderpoolAgent.image.tag=$(E2E_SPIDERPOOL_TAG) \ + --set spiderpoolController.image.registry="" \ + --set spiderpoolController.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \ + --set spiderpoolController.image.tag=$(E2E_SPIDERPOOL_TAG) \ + --set spiderpoolInit.image.registry="" \ + --set spiderpoolInit.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \ + --set spiderpoolInit.image.tag=$(E2E_SPIDERPOOL_TAG) " \ + helm --kubeconfig $(E2E_KUBECONFIG) upgrade $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool $${HELM_OPTION} \ + -n $(RELEASE_NAMESPACE) --wait || { KIND_CLUSTER_NAME=$(E2E_CLUSTER_NAME) ./scripts/debugEnv.sh $(E2E_KUBECONFIG) "detail"; exit 1; } + .PHONY: clean clean: @rm -rf $(CLUSTER_DIR)