From da460c23691660bc1c0ea4b2290e0790f4b1ff6c Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Fri, 24 Apr 2020 14:47:58 +0200 Subject: [PATCH] Add test command --- .github/workflows/build-command.yml | 1 + .github/workflows/command-dispatch.yml | 7 +- .github/workflows/test-command.yml | 166 +++++++++++++++++++++++++ tools/notify-job-status.py | 66 ++++++++++ 4 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-command.yml create mode 100755 tools/notify-job-status.py diff --git a/.github/workflows/build-command.yml b/.github/workflows/build-command.yml index 3d8e7043..9fad829c 100644 --- a/.github/workflows/build-command.yml +++ b/.github/workflows/build-command.yml @@ -1,4 +1,5 @@ # TODO: Don't build if the PR is not based on master's HEAD +# TODO: Only build the container if specific directories/files have changed, unless forced: https://coderwall.com/p/lz0uva/find-all-files-modified-between-commits-in-git name: build-command on: repository_dispatch: diff --git a/.github/workflows/command-dispatch.yml b/.github/workflows/command-dispatch.yml index 05973dbd..fe8fa895 100644 --- a/.github/workflows/command-dispatch.yml +++ b/.github/workflows/command-dispatch.yml @@ -9,7 +9,7 @@ jobs: - name: Slash Command Dispatch uses: peter-evans/slash-command-dispatch@v1 with: - token: ${{ secrets.TOKEN }} + token: ${{ secrets.EMBERIO_ACCESS_TOKEN }} issue-type: pull-request reactions: true # commands: help, build @@ -26,5 +26,10 @@ jobs: "command": "build", "permission": "write", "issue_type": "pull-request" + }, + { + "command": "test", + "permission": "write", + "issue_type": "pull-request" } ] diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml new file mode 100644 index 00000000..84892c17 --- /dev/null +++ b/.github/workflows/test-command.yml @@ -0,0 +1,166 @@ +# TODO: Don't build if the PR is not based on master's HEAD +# TODO: Only build the container if specific directories/files have changed, unless forced: https://coderwall.com/p/lz0uva/find-all-files-modified-between-commits-in-git +name: test-command +on: + repository_dispatch: + types: [test-command] +env: + AUTHORIZED: ${{ github.actor != github.event.client_payload.pull_request.base.repo.owner.login }} + DETAILS_URL: ${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }} + DOCKER_REPO: embercsi/ci_images + IMAGE_TAG: ${{ format('{0}-{1}', github.event.client_payload.pull_request.number, github.event.client_payload.pull_request.head.sha) }} + +jobs: + + build: + name: internal/build-pr-images + env: + JOB_NAME: internal/build-pr-images + TAGS_MISSING: false + + # In the future we may want to return embercsi/ember-csi:{master,latest} + outputs: + image_base_name: ${{ format('{0}:{1}-', env.DOCKER_REPO, env.IMAGE_TAG) }} + + runs-on: ubuntu-latest + steps: + # Checkout the pull request branch from the repository + - uses: actions/checkout@v2 + if: env.AUTHORIZED == 'true' + with: + token: ${{ secrets.TOKEN }} + repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Write comment + uses: peter-evans/create-or-update-comment@v1 + with: + token: ${{ secrets.EMBERIO_ACCESS_TOKEN }} + repository: ${{ github.event.client_payload.github.payload.repository.full_name }} + issue-number: ${{ github.event.client_payload.pull_request.number }} + # comment-id: ${{ github.event.client_payload.github.payload.comment.id }} + body: | + [Test workflow has started][1] + + [1]: ${{ env.DETAILS_URL }} + + - name: Notify in_progress + if: env.AUTHORIZED == 'true' + run: ./tools/notify-job-status.py ${{ env.JOB_NAME }} ${{ github.run_id }} pending ${{ github.repository }} ${{ github.event.client_payload.pull_request.head.sha }} + #run: | + # curl -X POST -H "Authorization: token ${{ secrets.TOKEN }}" -H "Content-Type: application/json" -d '{"context": "${{ env.JOB_NAME }}","state": "pending","target_url": "${{ env.DETAILS_URL }}"}' "https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.client_payload.pull_request.head.sha }}" + env: + TOKEN: ${{ secrets.TOKEN }} + + - name: Check if images exist + if: env.AUTHORIZED == 'true' + run: | + for i in 7 8; do + if ! curl -L --silent ${{ format('https://hub.docker.com/v2/repositories/{0}/tags/{1}', env.DOCKER_REPO, env.IMAGE_TAG) }}-$i | grep '"name"'; then + echo "::set-env name=TAGS_MISSING::true" + break + fi + done + + - name: Build images + if: env.AUTHORIZED == 'true' && env.TAGS_MISSING == 'true' + run: | + echo 'Start' + echo "::add-mask::${{ secrets.DOCKER_PASSWORD }}" + EMBER_VERSION=$IMAGE_TAG SOURCE_BRANCH=master hooks/build + echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + docker tag $DOCKER_REPO:master7 $DOCKER_REPO:${IMAGE_TAG}-7 + docker tag $DOCKER_REPO:master8 $DOCKER_REPO:${IMAGE_TAG}-8 + docker push $DOCKER_REPO:${IMAGE_TAG}-7 + docker push $DOCKER_REPO:${IMAGE_TAG}-8 + echo 'Done' + + # - name: Get result status + # if: always() && env.AUTHORIZED == 'true' + # run: | + # status=`echo "${{ job.status }}" | tr '[:upper:]' '[:lower:]'` + # if [[ "$status" == "cancelled" ]]; then + # echo "::set-env name=JOB_RESULT::error" + # else + # echo "::set-env name=JOB_RESULT::$status" + # fi + + - name: Notify result status + if: always() && env.AUTHORIZED == 'true' + run: ./tools/notify-job-status.py ${{ env.JOB_NAME }} ${{ github.run_id }} ${{ job.status }} ${{ github.repository }} ${{ github.event.client_payload.pull_request.head.sha }} + # run: | + # curl -X POST -H "Authorization: token ${{ secrets.TOKEN }}" -H "Content-Type: application/json" -d '{"context": "${{ env.JOB_NAME }}","state": "${{ env.JOB_RESULT }}","target_url": "${{ env.DETAILS_URL }}"}' "https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.client_payload.pull_request.head.sha }}" + env: + TOKEN: ${{ secrets.TOKEN }} + + + functional: + needs: build + strategy: + fail-fast: false + matrix: + centos: [7, 8] + backend: [lvm, ceph] + csi: [v1.1] + include: + - centos: 7 + backend: lvm + csi: v1.0 + - centos: 7 + backend: lvm + csi: v0.3 + - centos: 7 + backend: lvm + csi: v0.2 + name: "3rdparty/${{ matrix.backend }}-functional-centos${{ matrix.centos }}-csi_${{ matrix.csi }}" + runs-on: ${{ matrix.backend }} + # Fails: + # runs-on: [self-hosted, ${{ matrix.backend }}] + # runs-on: "[self-hosted, ${{ matrix.backend }}]" + # runs-on: ${{ format('[ self-hosted, {0} ]', matrix.backend) }} + # These work: + #runs-on: [self-hosted, lvm] + # runs-on: self-hosted + env: + JOB_NAME: "3rdparty/${{ matrix.backend }}-functional-centos${{ matrix.centos }}-csi_${{ matrix.csi }}" + BACKEND_NAME: ${{ matrix.backend }} + steps: + # Checkout the pull request branch from the repository + - uses: actions/checkout@v2 + if: env.AUTHORIZED == 'true' + with: + repository: embercsi/3rd-party-ci + ref: gh-actions + + - name: Notify in_progress + # run: ./ci-scripts/notify-job-status.py ${{ env.JOB_NAME }} pending ${{ env.DETAILS_URL }} ${{ github.repository }} ${{ github.event.client_payload.pull_request.head.sha }} + run: ./ci-scripts/notify-job-status.py ${{ env.JOB_NAME }} ${{ github.run_id }} pending ${{ github.repository }} ${{ github.event.client_payload.pull_request.head.sha }} + env: + TOKEN: ${{ secrets.TOKEN }} + + - name: Testing + run: | + pwd + export HOST_IP=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) + echo Running with backend ${{ matrix.backend }} on a Centos ${{ matrix.centos }} VM + ./ci-scripts/vm-run-functional.sh + env: + EMBER_IMAGE: ${{needs.build.outputs.image_base_name}} + CENTOS_VERSION: ${{matrix.centos}} + JOB_ID: ${{github.run_id}} + CSI_SPEC_VERSION: ${{matrix.csi}} + # BACKEND_NAME: ${{matrix.backend}} + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@v2-preview + with: + name: "${{ matrix.backend }}-functional-centos${{ matrix.centos }}-artifacts" + path: 'artifacts/*' + + - name: Notify result status + if: always() + # run: ./ci-scripts/notify-job-status.py ${{ env.JOB_NAME }} ${{ job.status }} ${{ env.DETAILS_URL }} ${{ github.repository }} + run: ./ci-scripts/notify-job-status.py ${{ env.JOB_NAME }} ${{ github.run_id }} ${{ job.status }} ${{ github.repository }} ${{ github.event.client_payload.pull_request.head.sha }} + env: + TOKEN: ${{ secrets.TOKEN }} diff --git a/tools/notify-job-status.py b/tools/notify-job-status.py new file mode 100755 index 00000000..34f3a625 --- /dev/null +++ b/tools/notify-job-status.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +import json +import os +import sys +import urllib2 + +ERNO_NUMBER_ARGS = 1 +ERNO_JOB_INFO = 2 +ERNO_STATUS = 3 + +HEADERS = {'Authorization': 'token ' + os.environ['TOKEN'], + 'Content-Type': 'application/json', + 'User-Agent': 'Ember-CSI_CI'} + + +def get_job_url(repository, job_name, run_id): + url = 'https://api.github.com/repos/%s/actions/runs/%s/jobs' % (repository, + run_id) + req = urllib2.Request(url, headers=HEADERS) + response = urllib2.urlopen(req) + result = json.loads(response.read()) + for job in result['jobs']: + if job['name'] != job_name: + continue + return job['html_url'] + '?check_suite_focus=true' + sys.stderr.write('Could not find the requested job') + exit(ERNO_JOB_INFO) + + +def set_state(state, repository, job_name, target_url, commit_sha): + url = 'https://api.github.com/repos/%s/statuses/%s' % (repository, + commit_sha) + data = json.dumps({'context': job_name, + 'state': state, + 'target_url': target_url}) + + req = urllib2.Request(url, data, HEADERS) + error = None + try: + response = urllib2.urlopen(req) + if response.code != 201: + error = 'Status code is %s' % response.code + except Exception as exc: + error = str(exc) + + if error: + sys.stderr.write('Error sending status change: %s\n' % error) + exit(ERNO_STATUS) + + +if __name__ == '__main__': + if len(sys.argv) != 6: + sys.stderr.write('Wrong number of arguments:\n\t%s job_name run_id ' + 'state repository commit_sha\n' % + os.path.basename(sys.argv[0])) + exit(ERNO_NUMBER_ARGS) + + __, job_name, run_id, state, repository, commit_sha = sys.argv + + # Convert gh-actions job status to old checks status + state = state.lower() + if state == 'cancelled': + state = 'error' + + target_url = get_job_url(repository, job_name, run_id) + set_state(state, repository, job_name, target_url, commit_sha)