Auto Upgrade Test #22
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Auto Upgrade Test | |
| permissions: write-all | |
| env: | |
| CLUSTER_NAME: spider | |
| E2E_TIME_OUT: 60m | |
| KUBECONFIG_PATH: /home/runner/work/spiderpool/spiderpool/test/.cluster | |
| on: | |
| schedule: | |
| - cron: "0 20 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| ref: | |
| description: 'sha, tag, branch. also this is the version you will upgrade to.' | |
| required: true | |
| default: main | |
| e2e_enabled: | |
| description: 'run e2e test' | |
| required: false | |
| type: choice | |
| default: "false" | |
| options: | |
| - "true" | |
| - "false" | |
| jobs: | |
| get_ref: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| ref: ${{ env.RUN_REF }} | |
| e2e_enabled: ${{ env.RUN_E2E_ENABLED }} | |
| steps: | |
| - name: Get Ref | |
| id: get_ref | |
| run: | | |
| if ${{ github.event_name == 'workflow_dispatch' }} ; then | |
| echo "call by self workflow_dispatch" | |
| echo "RUN_TAG=${{ github.event.inputs.ref }}" >> $GITHUB_ENV | |
| if ${{ github.event.inputs.e2e_enabled == 'true' }}; then | |
| echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV | |
| else | |
| echo "RUN_E2E_ENABLED=false" >> $GITHUB_ENV | |
| fi | |
| else | |
| # schedule event | |
| # use main sha for ci image tag | |
| echo "trigger by schedule" | |
| echo "RUN_TAG=main" >> $GITHUB_ENV | |
| echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV | |
| echo "DEFAULT_SPIDERPOOL_VERSION=true" >> $GITHUB_ENV | |
| fi | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ env.RUN_TAG }} | |
| - name: Result Ref | |
| id: result | |
| run: | | |
| ref=$( git show -s --format='format:%H') | |
| echo "RUN_REF=${ref}" >> $GITHUB_ENV | |
| call_build_ci_image: | |
| needs: [get_ref] | |
| uses: ./.github/workflows/build-image-ci.yaml | |
| with: | |
| ref: ${{ needs.get_ref.outputs.ref }} | |
| push: false | |
| secrets: inherit | |
| lint_chart_against_release_image: | |
| needs: get_ref | |
| uses: ./.github/workflows/call-lint-chart.yaml | |
| with: | |
| ref: ${{ needs.get_ref.outputs.ref }} | |
| secrets: inherit | |
| trivy_scan_images: | |
| needs: [call_build_ci_image, get_ref] | |
| uses: ./.github/workflows/trivy-scan-image.yaml | |
| with: | |
| image_tag: ${{ needs.call_build_ci_image.outputs.imageTag }} | |
| ref: ${{ needs.get_ref.outputs.ref }} | |
| secrets: inherit | |
| run_upgrade_tests_on_release_version: | |
| needs: [call_build_ci_image, get_ref] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Synchronise with the latest releases of each version | |
| version: [release-v0.7, release-v0.8, release-v0.9] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ matrix.version }} | |
| - name: Get image tag | |
| run: | | |
| cat VERSION | |
| imageTag=$(grep -E "^[[:space:]]*v[0-9]+.[0-9]+.[0-9]+[[:space:]]*$" VERSION) | |
| echo "imageTag=$imageTag" >> $GITHUB_ENV | |
| - name: Prepare | |
| id: prepare | |
| run: | | |
| echo "ref: ${{ inputs.ref }} " | |
| echo "===== image " | |
| echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller-ci:${{ env.imageTag }}" | |
| echo "ci race image tag: ghcr.io/${{ github.repository }}/spiderpool-controller-ci:${{ env.imageTag }}-race" | |
| echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-agent-ci:${{ env.imageTag }}" | |
| echo "ci race image tag: ghcr.io/${{ github.repository }}/spiderpool-agent-ci:${{ env.imageTag }}-race" | |
| TMP=` date +%m%d%H%M%S ` | |
| E2E_CLUSTER_NAME="spiderpool${TMP}" | |
| echo "E2E_CLUSTER_NAME=${E2E_CLUSTER_NAME}" >> $GITHUB_ENV | |
| - name: Setup Golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: 1.21.4 | |
| # https://github.com/helm/kind-action | |
| - name: Install Kind Bin | |
| uses: helm/[email protected] | |
| with: | |
| install_only: true | |
| - name: Install Tools | |
| run: | | |
| # install kind/p2ctl/helm/ginkgo | |
| bash ./test/scripts/install-tools.sh | |
| # test against commit version | |
| # https://github.com/kubernetes-sigs/kind/issues/2863 | |
| - name: Setup Kind Cluster on ${{ matrix.version }} | |
| uses: nick-invision/retry@v2 | |
| with: | |
| timeout_minutes: 20 | |
| max_attempts: 3 | |
| command: | | |
| make e2e_init_underlay -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \ | |
| -e E2E_SPIDERPOOL_TAG=${{ env.imageTag }} \ | |
| -e PYROSCOPE_LOCAL_PORT="" \ | |
| -e INSTALL_KUBEVIRT=true \ | |
| -e INSTALL_KRUISE=true \ | |
| -e INSTALL_KDOCTOR=true \ | |
| -e INSTALL_OVS=true \ | |
| -e INSTALL_RDMA=true \ | |
| -e INSTALL_SRIOV=true | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| # this might remove tools that are actually needed, | |
| # if set to "true" but frees about 6 GB | |
| tool-cache: false | |
| # all of these default to true, but feel free to set to | |
| # "false" if necessary for your workflow | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| - name: Move kubeconfig | |
| run: | | |
| cp -r ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config /tmp | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ needs.get_ref.outputs.ref }} | |
| - name: Move kubeconfig to branch ${{ needs.get_ref.outputs.ref }} | |
| run: | | |
| mkdir -p ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/ | |
| cp -r /tmp/config ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config | |
| - name: Download spiderpool-agent image | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: image-tar-spiderpool-agent | |
| path: test/.download | |
| - name: Download spiderpool-controller image | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: image-tar-spiderpool-controller | |
| path: test/.download | |
| - name: Load Images | |
| run: | | |
| TAR_FILES=` ls test/.download ` | |
| echo $TAR_FILES | |
| for ITEM in $TAR_FILES ; do | |
| IMAGE_NAME=${ITEM%*.tar} | |
| echo ${IMAGE_NAME} | |
| cat test/.download/${ITEM} | docker import - ${IMAGE_NAME}:${{ needs.call_build_ci_image.outputs.imageTag }} | |
| done | |
| - name: Upgrade to version ${{ needs.get_ref.outputs.ref }} | |
| continue-on-error: true | |
| run: | | |
| make upgrade_e2e_spiderpool -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \ | |
| -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race \ | |
| -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \ | |
| - name: Run e2e Test on ${{ needs.get_ref.outputs.ref }} | |
| id: run_e2e | |
| continue-on-error: true | |
| if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} | |
| run: | | |
| RESULT=0 | |
| make e2e_test_underlay -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \ | |
| -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }} || RESULT=1 | |
| if ((RESULT==0)) ; then | |
| echo "RUN_E2E_PASS=true" >> $GITHUB_ENV | |
| else | |
| echo "RUN_E2E_PASS=false" >> $GITHUB_ENV | |
| fi | |
| if [ -f "test/e2edebugLog" ] ; then | |
| echo "UPLOAD_E2E_LOG=true" >> $GITHUB_ENV | |
| else | |
| echo "UPLOAD_E2E_LOG=false" >> $GITHUB_ENV | |
| fi | |
| if [ -f "./e2ereport.json" ] ; then | |
| echo "error, did not find e2e report" | |
| echo "UPLOAD_E2E_REPORT=true" >> $GITHUB_ENV | |
| else | |
| echo "UPLOAD_E2E_REPORT=false" >> $GITHUB_ENV | |
| fi | |
| - name: Upload e2e log | |
| if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: ${{ needs.get_ref.outputs.ref }}-debuglog.txt | |
| path: test/e2edebugLog.txt | |
| retention-days: 7 | |
| - name: Upload e2e report | |
| if: ${{ env.UPLOAD_E2E_REPORT == 'true' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: ${{ needs.get_ref.outputs.ref }}-e2ereport.json | |
| path: e2ereport.json | |
| retention-days: 1 | |
| creat_issue: | |
| runs-on: ubuntu-latest | |
| needs: [run_upgrade_tests_on_release_version] | |
| if: ${{ always() && needs.run_upgrade_tests_on_release_version.result == 'failure' }} | |
| steps: | |
| - name: echo | |
| run: | | |
| echo ${{ github.repository }} | |
| echo ${{ github.repository_owner }} | |
| echo "TIMESTAMP=`date +%Y-%m-%d`" >> $GITHUB_ENV | |
| - name: create an issue | |
| uses: dacbd/[email protected] | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| title: "Nightly upgrade test CI ${{ ENV.TIMESTAMP }}: Failed" | |
| body: | | |
| action url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| labels: "kind/ci-bug" | |
| assignees: "Icarus9913,ty-dc" |