diff --git a/.github/workflows/checks-workflows.yml b/.github/workflows/checks-workflows.yml new file mode 100644 index 000000000..9b034dba4 --- /dev/null +++ b/.github/workflows/checks-workflows.yml @@ -0,0 +1,29 @@ +name: Checks for GitHub workflows + +on: + push: + branches: [main] + pull_request: + branches: + - main + +env: + PYTHONUNBUFFERED: 1 + FORCE_COLOR: 1 + PYTHON_VERSION: "3.11" + +jobs: + checks-workflows: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Checks for GitHub workflows + run: | + python tools/scan_yaml_for_risky_text.py .github/workflows diff --git a/.github/workflows/circleci-trigger.yml b/.github/workflows/circleci-trigger.yml deleted file mode 100644 index 9b8713c48..000000000 --- a/.github/workflows/circleci-trigger.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: CircleCI tests trigger - -on: - push: - branches: [main] - pull_request: - branches: - - main - -env: - PYTHONUNBUFFERED: 1 - FORCE_COLOR: 1 - -jobs: - circleci-trigger-fork: - if: ${{ github.event.pull_request.head.repo.fork }} - name: CircleCI tests trigger - runs-on: ubuntu-latest - steps: - - name: Passed fork step - run: echo "Success!" - - circleci-trigger: - if: ${{ ! github.event.pull_request.head.repo.fork }} - name: CircleCI tests trigger - runs-on: ubuntu-latest - steps: - - name: Start CircleCI pipeline - run: | - create_circleci_pipeline() { - local branch=$1 - - local json_data=$(jq -n --arg branch "$branch" --arg vizro_branch "${{ github.head_ref }}" '{branch: $branch, parameters: {branch: $branch, vizro_branch: $vizro_branch}}') - - curl --silent --request POST \ - --url "${{ secrets.QA_PIPELINE_URL }}" \ - --header "Circle-Token: ${{ secrets.CIRCLECI_API_KEY }}" \ - --header "content-type: application/json" \ - --data "$json_data" \ - | jq -r '.id' - } - - PIPELINE=$(create_circleci_pipeline "${{ github.head_ref }}") - - # If the above returns null then the QA repo doesn't contain current dev branch, so we use main branch. - if [[ "$PIPELINE" == "null" ]]; then - PIPELINE=$(create_circleci_pipeline "main") - fi - echo "Started pipeline with id $PIPELINE" - - echo "PIPELINE=$PIPELINE" >> $GITHUB_ENV - - - name: Wait for pipeline to run - run: sleep 60 - - - name: Check pipeline status - run: | - get_pipeline_status() { - curl --silent --request GET \ - --url "https://circleci.com/api/v2/pipeline/$PIPELINE/workflow" \ - --header "Circle-Token: ${{ secrets.CIRCLECI_API_KEY }}" \ - --header "content-type: application/json" \ - | jq -r '.items[0].status' - } - - while pipeline_status=$(get_pipeline_status); [[ "$pipeline_status" == "running" ]]; do - echo $pipeline_status - sleep 15 - done - - if [[ "$pipeline_status" != "success" ]]; then - echo "Pipeline not completed successfully - status was ${pipeline_status}" - exit 1 - fi diff --git a/.github/workflows/test-integration-vizro-ai.yml b/.github/workflows/test-integration-vizro-ai.yml index bb5917e16..758cd04eb 100644 --- a/.github/workflows/test-integration-vizro-ai.yml +++ b/.github/workflows/test-integration-vizro-ai.yml @@ -93,3 +93,16 @@ jobs: cd ../vizro-ai hatch run ${{ matrix.hatch-env }}:pip install ../vizro-core/dist/vizro*.tar.gz hatch run ${{ matrix.hatch-env }}:test-integration + + - name: Send custom JSON data to Slack + id: slack + uses: slackapi/slack-github-action@v1.26.0 + if: failure() + with: + payload: | + { + "text": "Vizro-ai ${{ matrix.hatch-env }} integration tests build result: ${{ job.status }}\nBranch: ${{ github.head_ref }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/vizro-qa-tests-trigger.yml b/.github/workflows/vizro-qa-tests-trigger.yml new file mode 100644 index 000000000..1d223a6de --- /dev/null +++ b/.github/workflows/vizro-qa-tests-trigger.yml @@ -0,0 +1,50 @@ +name: Vizro QA tests trigger + +on: + push: + branches: [main] + pull_request: + branches: + - main + +env: + PYTHONUNBUFFERED: 1 + FORCE_COLOR: 1 + +jobs: + vizro-qa-test-trigger-fork: + if: ${{ github.event.pull_request.head.repo.fork }} + name: Vizro QA ${{ matrix.label }} trigger + runs-on: ubuntu-latest + strategy: + matrix: + include: + - label: integration tests + - label: notebooks tests + steps: + - name: Passed fork step + run: echo "Success!" + + vizro-qa-tests-trigger: + if: ${{ ! github.event.pull_request.head.repo.fork }} + name: Vizro QA ${{ matrix.label }} trigger + runs-on: ubuntu-latest + strategy: + matrix: + include: + - label: integration tests + - label: notebooks test + steps: + - uses: actions/checkout@v4 + - name: Tests trigger + run: | + export INPUT_OWNER=${{ secrets.VIZRO_QA_ORG }} + export INPUT_REPO=${{ secrets.VIZRO_QA_REPO }} + if [ "${{ matrix.label }}" == "integration tests" ]; then + export INPUT_WORKFLOW_FILE_NAME=${{ secrets.VIZRO_QA_INTEGRATION_TESTS_WORKFLOW }} + elif [ "${{ matrix.label }}" == "notebooks test" ]; then + export INPUT_WORKFLOW_FILE_NAME=${{ secrets.VIZRO_QA_NOTEBOOKS_TESTS_WORKFLOW }} + fi + export INPUT_GITHUB_TOKEN=${{ secrets.VIZRO_SVC_PAT }} + export INPUT_REF=${{ github.head_ref }} + tools/trigger-workflow-and-wait.sh diff --git a/tools/scan_yaml_for_risky_text.py b/tools/scan_yaml_for_risky_text.py new file mode 100644 index 000000000..70240f872 --- /dev/null +++ b/tools/scan_yaml_for_risky_text.py @@ -0,0 +1,19 @@ +"""Check for security issues in workflows files.""" + +import sys +from pathlib import Path + +# according to this article: https://nathandavison.com/blog/github-actions-and-the-threat-of-malicious-pull-requests +# we should avoid using `pull_request_target` for security reasons +risky_text = "pull_request_target" + + +def find_risky_files(path: str): + """Searching for risky text in yml files for given path.""" + return {file for file in Path(path).rglob("*.yml") if risky_text in file.read_text()} + + +if __name__ == "__main__": + risky_files = find_risky_files(sys.argv[1]) + if risky_files: + sys.exit(f"{risky_text} found in files {risky_files}.") diff --git a/tools/trigger-workflow-and-wait.sh b/tools/trigger-workflow-and-wait.sh new file mode 100755 index 000000000..4447efb65 --- /dev/null +++ b/tools/trigger-workflow-and-wait.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash + +#MIT License +# +#Copyright (c) 2020 Convictional, Inc. +# +#Permission is hereby granted, free of charge, to any person obtaining a copy +#of this software and associated documentation files (the "Software"), to deal +#in the Software without restriction, including without limitation the rights +#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the Software is +#furnished to do so, subject to the following conditions: +# +#The above copyright notice and this permission notice shall be included in all +#copies or substantial portions of the Software. +# +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +#SOFTWARE. + +set -e + +GITHUB_API_URL="${API_URL:-https://api.github.com}" +GITHUB_SERVER_URL="${SERVER_URL:-https://github.com}" + +validate_args() { + wait_interval=10 # Waits for 10 seconds + if [ "${INPUT_WAIT_INTERVAL}" ] + then + wait_interval=${INPUT_WAIT_INTERVAL} + fi + + propagate_failure=true + if [ -n "${INPUT_PROPAGATE_FAILURE}" ] + then + propagate_failure=${INPUT_PROPAGATE_FAILURE} + fi + + trigger_workflow=true + if [ -n "${INPUT_TRIGGER_WORKFLOW}" ] + then + trigger_workflow=${INPUT_TRIGGER_WORKFLOW} + fi + + wait_workflow=true + if [ -n "${INPUT_WAIT_WORKFLOW}" ] + then + wait_workflow=${INPUT_WAIT_WORKFLOW} + fi + + if [ -z "${INPUT_OWNER}" ] + then + echo "Error: Owner is a required argument." + exit 1 + fi + + if [ -z "${INPUT_REPO}" ] + then + echo "Error: Repo is a required argument." + exit 1 + fi + + if [ -z "${INPUT_GITHUB_TOKEN}" ] + then + echo "Error: Github token is required. You can head over settings and" + echo "under developer, you can create a personal access tokens. The" + echo "token requires repo access." + exit 1 + fi + + if [ -z "${INPUT_WORKFLOW_FILE_NAME}" ] + then + echo "Error: Workflow File Name is required" + exit 1 + fi + + client_payload=$(echo '{}' | jq -c) + if [ "${INPUT_CLIENT_PAYLOAD}" ] + then + client_payload=$(echo "${INPUT_CLIENT_PAYLOAD}" | jq -c) + fi + + ref="main" + if [ "$INPUT_REF" ] + then + ref="${INPUT_REF}" + fi +} + +lets_wait() { + echo "Sleeping for ${wait_interval} seconds" + sleep "$wait_interval" +} + +api() { + path=$1; shift + if response=$(curl --fail-with-body -sSL \ + "${GITHUB_API_URL}/repos/${INPUT_OWNER}/${INPUT_REPO}/actions/$path" \ + -H "Authorization: Bearer ${INPUT_GITHUB_TOKEN}" \ + -H 'Accept: application/vnd.github.v3+json' \ + -H 'Content-Type: application/json' \ + "$@") + then + echo "$response" + else + echo >&2 "api failed:" + echo >&2 "path: $path" + echo >&2 "response: $response" + if [[ "$response" == *'"Server Error"'* ]]; then + echo "Server error - trying again" + else + exit 1 + fi + fi +} + +lets_wait() { + local interval=${1:-$wait_interval} + echo >&2 "Sleeping for $interval seconds" + sleep "$interval" +} + +# Return the ids of the most recent workflow runs, optionally filtered by user +get_workflow_runs() { + since=${1:?} + + query="event=workflow_dispatch&created=>=$since${INPUT_GITHUB_USER+&actor=}${INPUT_GITHUB_USER}&per_page=100" + + echo "Getting workflow runs using query: ${query}" >&2 + + api "workflows/${INPUT_WORKFLOW_FILE_NAME}/runs?${query}" | + jq -r '.workflow_runs[].id' | + sort # Sort to ensure repeatable order, and lexicographically for compatibility with join +} + +trigger_workflow() { + START_TIME=$(date +%s) + SINCE=$(date -u -Iseconds -d "@$((START_TIME - 120))") # Two minutes ago, to overcome clock skew + + OLD_RUNS=$(get_workflow_runs "$SINCE") + + echo >&2 "Triggering workflow:" + echo >&2 " workflows/${INPUT_WORKFLOW_FILE_NAME}/dispatches" + echo >&2 " {\"ref\":\"${ref}\",\"inputs\":${client_payload}}" + + api "workflows/${INPUT_WORKFLOW_FILE_NAME}/dispatches" \ + --data "{\"ref\":\"${ref}\",\"inputs\":${client_payload}}" + + NEW_RUNS=$OLD_RUNS + while [ "$NEW_RUNS" = "$OLD_RUNS" ] + do + lets_wait + NEW_RUNS=$(get_workflow_runs "$SINCE") + done + + # Return new run ids + join -v2 <(echo "$OLD_RUNS") <(echo "$NEW_RUNS") +} + +comment_downstream_link() { + if response=$(curl --fail-with-body -sSL -X POST \ + "${INPUT_COMMENT_DOWNSTREAM_URL}" \ + -H "Authorization: Bearer ${INPUT_COMMENT_GITHUB_TOKEN}" \ + -H 'Accept: application/vnd.github.v3+json' \ + -d "{\"body\": \"Running downstream job at $1\"}") + then + echo "$response" + else + echo >&2 "failed to comment to ${INPUT_COMMENT_DOWNSTREAM_URL}:" + fi +} + +wait_for_workflow_to_finish() { + last_workflow_id=${1:?} + last_workflow_url="${GITHUB_SERVER_URL}/${INPUT_OWNER}/${INPUT_REPO}/actions/runs/${last_workflow_id}" + + echo "Waiting for workflow to finish:" + echo "The workflow id is [${last_workflow_id}]." + echo "The workflow logs can be found at ${last_workflow_url}" + echo "workflow_id=${last_workflow_id}" >> $GITHUB_OUTPUT + echo "workflow_url=${last_workflow_url}" >> $GITHUB_OUTPUT + echo "" + + if [ -n "${INPUT_COMMENT_DOWNSTREAM_URL}" ]; then + comment_downstream_link ${last_workflow_url} + fi + + conclusion=null + status= + + while [[ "${conclusion}" == "null" && "${status}" != "completed" ]] + do + lets_wait + + workflow=$(api "runs/$last_workflow_id") + conclusion=$(echo "${workflow}" | jq -r '.conclusion') + status=$(echo "${workflow}" | jq -r '.status') + + echo "Checking conclusion [${conclusion}]" + echo "Checking status [${status}]" + echo "conclusion=${conclusion}" >> $GITHUB_OUTPUT + done + + if [[ "${conclusion}" == "success" && "${status}" == "completed" ]] + then + echo "Yes, success" + else + # Alternative "failure" + echo "Conclusion is not success, it's [${conclusion}]." + + if [ "${propagate_failure}" = true ] + then + echo "Propagating failure to upstream job" + exit 1 + fi + fi +} + +main() { + validate_args + + if [ "${trigger_workflow}" = true ] + then + run_ids=$(trigger_workflow) + else + echo "Skipping triggering the workflow." + fi + + if [ "${wait_workflow}" = true ] + then + for run_id in $run_ids + do + wait_for_workflow_to_finish "$run_id" + done + else + echo "Skipping waiting for workflow." + fi +} + +main