diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 6e130a8144..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: 2.1 - -parameters: - release_kedro: - type: boolean - default: false - # The parameters below are set in CircleCI UI. - # https://app.circleci.com/settings/project/github/kedro-org/kedro/triggers?return-to=https%3A%2F%2Fapp.circleci.com%2Fpipelines%2Fgithub%2Fkedro-org%2Fkedro&triggerSource=&scheduledTriggerId=61f7226f-f092-4449-98d9-40420f8c46b2&success=true - run_hourly: - type: boolean - default: false - run_nightly: - type: boolean - default: false - -setup: true - -orbs: - path-filtering: circleci/path-filtering@0.1.1 - -workflows: - regular: - jobs: - # the path-filtering/filter job determines which pipeline - # parameters to update, i.e. which builds to run. - - path-filtering/filter: - name: check-updated-files - base-revision: main - config-path: .circleci/continue_config.yml - # - mapping: | - docs/.* docs_change true - ^((?!docs/).)*$ code_change true diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml deleted file mode 100644 index 223367058a..0000000000 --- a/.circleci/continue_config.yml +++ /dev/null @@ -1,164 +0,0 @@ -version: 2.1 - - -# the default pipeline parameters, which will be updated according to -# the results of the path-filtering orb -parameters: - code_change: - type: boolean - default: false - docs_change: - type: boolean - default: false - release_kedro: - type: boolean - default: false - # The parameters below are set in CircleCI UI. - # https://app.circleci.com/settings/project/github/kedro-org/kedro/triggers?return-to=https%3A%2F%2Fapp.circleci.com%2Fpipelines%2Fgithub%2Fkedro-org%2Fkedro&triggerSource=&scheduledTriggerId=61f7226f-f092-4449-98d9-40420f8c46b2&success=true - run_hourly: - type: boolean - default: false - run_nightly: - type: boolean - default: false - -orbs: - win: circleci/windows@2.4.1 - -# No windows executor is listed here since windows builds use win/default and modify -# the Python version through the conda environment. -executors: - docker: - parameters: - python_version: - type: string - docker: - - image: public.ecr.aws/g0x0s3o2/kedro-builder:<> - resource_class: medium+ - -commands: - setup_conda: - steps: - - run: - name: Run conda.sh - command: echo ". /home/circleci/miniconda/etc/profile.d/conda.sh" >> $BASH_ENV - - run: - name: Activate conda environment - command: echo "conda deactivate; conda activate kedro_builder" >> $BASH_ENV - - setup_requirements: - steps: - - run: - name: Install venv for some pre-commit hooks - command: conda install -y virtualenv - - run: - # pytables does not work properly with python 3.9 to handle our HDFDataset - # if pip-installed, so we install this dependency via conda - name: Install pytables - command: conda install -c conda-forge pytables -y - - run: - name: Install requirements and test requirements - command: pip install --upgrade .[test] - - run: - # this is needed to fix java cacerts so - # spark can automatically download packages from mvn - # https://stackoverflow.com/a/50103533/1684058 - name: Fix cacerts - command: | - sudo rm /etc/ssl/certs/java/cacerts - sudo update-ca-certificates -f - - run: - # Since recently Spark installation for some reason does not have enough permissions to execute - # /home/circleci/miniconda/envs/kedro_builder/lib/python3.X/site-packages/pyspark/bin/spark-class. - # So fixing it manually here. - name: Fix Spark permissions - command: sudo chmod -R u+x /home/circleci/miniconda/envs/kedro_builder/lib/ - - run: - name: Print Python environment - command: make print-python-env - - run: - name: Pip freeze - command: pip freeze - - setup: - steps: - - checkout - - setup_conda - - setup_requirements - -jobs: - sync: - docker: - # https://circleci.com/docs/2.0/circleci-images/#circleci-base-image - - image: cimg/base:2020.01 - steps: - - checkout - - add_ssh_keys - - run: - name: Set git email and name - command: | - git config --global user.email "kedro@kedro.com" - git config --global user.name "Kedro" - - run: - name: Trigger Read The Docs build - command: ./tools/circleci/rtd-build.sh ${RTD_TOKEN} latest - - run: - name: Maybe merge main into develop or raise a PR - command: ./tools/circleci/github_scripts/merge.sh . "main" "develop" "${GITHUB_TAGGING_TOKEN}" - - merge_pr_to_develop: - docker: - # https://circleci.com/docs/2.0/circleci-images/#circleci-base-image - - image: cimg/base:2020.01 - steps: - - checkout - - add_ssh_keys - - run: - name: Maybe merge an automatic PR into develop - command: ./tools/circleci/github_scripts/attempt_merge_pr.sh "merge-main-to-develop" "develop" "${GITHUB_TAGGING_TOKEN}" - - - # Trigger kedro-viz build to ensure tests in that project pass - viz_build: - docker: - - image: spotify/alpine # for bash and curl - steps: - - run: - name: Trigger kedro-viz build - command: | - curl --location --request POST \ - --url https://circleci.com/api/v2/project/github/kedro-org/kedro-viz/pipeline \ - --header "Circle-Token: $CIRCLE_VIZ_BUILD_TOKEN" \ - --header 'content-type: application/json' \ - --data '{"branch":"main"}' - -workflows: - version: 2.1 - - main_updated: - when: - and: - - not: <> - - not: <> - - not: <> - jobs: - - sync: - filters: - branches: - only: main - - viz_build: - filters: - branches: - only: main - - hourly_pr_merge: - when: - and: - - <> - - not: <> - - not: <> - jobs: - - merge_pr_to_develop: - filters: - branches: - only: main diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000000..9067247640 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,3 @@ +automerge: + - any: + - head-branch: ['merge-main-to-develop'] diff --git a/.github/workflows/auto-merge-prs.yml b/.github/workflows/auto-merge-prs.yml new file mode 100644 index 0000000000..eb24a40247 --- /dev/null +++ b/.github/workflows/auto-merge-prs.yml @@ -0,0 +1,20 @@ +name: Auto-merge sync PR +# This workflow automatically merges the PRs labelled "automerge". This is a part of syncing +# of `develop` and `main` branches. + +on: + schedule: + - cron: '*/60 * * * *' # every 60 minutes + workflow_dispatch: # on button click + +jobs: + automerge: + permissions: + pull-requests: read + contents: write + runs-on: ubuntu-latest + steps: + - name: Automerge PRs with "automerge" label + uses: "pascalgn/automerge-action@v0.16.2" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/label-pr.yml b/.github/workflows/label-pr.yml new file mode 100644 index 0000000000..b246dea118 --- /dev/null +++ b/.github/workflows/label-pr.yml @@ -0,0 +1,22 @@ +name: Label PRs +# This workflow is a part of the sync and merge job, it labels the PRs created by `sync.yml` +# workflows with the "automerge" label. This label is used by the `auto-merge-prs.yml` workflow + +on: + pull_request: + types: + - opened + - synchronize + branches: + - develop + + +jobs: + label-pr: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Label PR with "automerge" + uses: actions/labeler@v5 diff --git a/.github/workflows/merge-gatekeeper.yml b/.github/workflows/merge-gatekeeper.yml index 6d3e82eac9..40edde5918 100644 --- a/.github/workflows/merge-gatekeeper.yml +++ b/.github/workflows/merge-gatekeeper.yml @@ -24,4 +24,4 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} timeout: 3600 interval: 30 - ignored: 'lint_only, runner / vale,ci/circleci: win_e2e_tests-3.7,ci/circleci: win_pip_compile-3.9,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_pip_compile-3.8,ci/circleci: lint-3.7,ci/circleci: win_pip_compile-3.7,ci/circleci: pip_compile-3.7,ci/circleci: e2e_tests-3.7,ci/circleci: win_unit_tests-3.7,ci/circleci: win_unit_tests-3.9,ci/circleci: e2e_tests-3.8,ci/circleci: win_unit_tests-3.10,ci/circleci: win_pip_compile-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: unit_tests-3.9,ci/circleci: unit_tests-3.7,ci/circleci: win_e2e_tests-3.10,ci/circleci: pip_compile-3.8,ci/circleci: pip_compile-3.10,ci/circleci: win_e2e_tests-3.8,ci/circleci: lint-3.9,ci/circleci: pip_compile-3.9,ci/circleci: lint-3.10,build_code,ci/circleci: check-updated-files,regular' + ignored: 'CircleCI Pipeline, runner / vale,' diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml new file mode 100644 index 0000000000..eef98830b2 --- /dev/null +++ b/.github/workflows/sync.yml @@ -0,0 +1,25 @@ +name: Sync `main` and `develop` +# This workflow merges commits from `main` branch to `develop` branch if the commits can be merged +# cleanly, otherwise creates a PR for the maintainers to manually resolve the conflicts. + +on: + schedule: + - cron: '*/30 * * * *' # every 30 minutes + workflow_dispatch: # on button click + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set git email and name + run: | + git config --global user.email "kedro@kedro.com" + git config --global user.name "Kedro" + + - name: Maybe merge to develop or raise a PR + run: ./tools/github_actions/merge.sh . "main" "develop" "${{ secrets.GITHUB_TOKEN }}" diff --git a/tools/circleci/docker_build_img/Dockerfile b/tools/circleci/docker_build_img/Dockerfile deleted file mode 100644 index 8f0c7313a9..0000000000 --- a/tools/circleci/docker_build_img/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM cimg/python:3.8 - -WORKDIR /home/circleci - -RUN sudo apt-get update && \ - sudo apt-get install curl pandoc openjdk-8-jdk-headless -y && \ - sudo apt-get clean && \ - sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java - -# Update cacerts: https://stackoverflow.com/a/50103533/1684058 -RUN sudo rm /etc/ssl/certs/java/cacerts && \ - sudo update-ca-certificates -f - -RUN curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - > miniconda.sh && bash miniconda.sh -b -p /home/circleci/miniconda - -RUN sudo rm -rf ~/.pyenv/ /opt/circleci/.pyenv/ - -ARG PIP_REQS -ARG PY_VERSION -ARG CONDA_ENV_NAME=kedro_builder - -# Install/Setup anaconda env -RUN bash -c "source /home/circleci/miniconda/etc/profile.d/conda.sh && \ - echo \"$PIP_REQS\" > /tmp/requirements.txt && \ - conda create --name=$CONDA_ENV_NAME python=$PY_VERSION && \ - conda activate $CONDA_ENV_NAME && \ - pip install --no-cache-dir --prefer-binary --upgrade -r /tmp/requirements.txt" diff --git a/tools/circleci/docker_build_img/build.sh b/tools/circleci/docker_build_img/build.sh deleted file mode 100755 index f45404c4c2..0000000000 --- a/tools/circleci/docker_build_img/build.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -set -e -cd "$(dirname "$0")" - -KEDRO_REPO=$1 -ECR_IMAGE_URL=$2 -PY_VERSION=$3 - -get_pip_reqs() { - local project_path=$1 - cat $project_path/*requirements.txt | grep -v requirements -} - -docker_build() { - local pip_reqs="$1" - local image=$ECR_IMAGE_URL:$PY_VERSION - echo "Building docker image: $image" - docker build -t $image \ - --build-arg PIP_REQS="$pip_reqs" \ - --build-arg PY_VERSION=$PY_VERSION \ - . -} - -docker_push() { - local image=$ECR_IMAGE_URL:$PY_VERSION - echo "Pushing docker image: $image" - docker push $image -} - -main() { - local pip_reqs="$(get_pip_reqs $KEDRO_REPO)" - docker_build "$pip_reqs" - docker_push -} - -main diff --git a/tools/circleci/github_scripts/attempt_merge_pr.sh b/tools/circleci/github_scripts/attempt_merge_pr.sh deleted file mode 100755 index a9553e326b..0000000000 --- a/tools/circleci/github_scripts/attempt_merge_pr.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env bash -# Script to attempt to merge an automatic PR from main to develop. - -# Exit script if you try to use an uninitialized variable. -set -o nounset - -# Exit script if a statement returns a non-true return value. -set -o errexit - -SOURCE_BRANCH=$1 -TARGET_BRANCH=$2 -GITHUB_TAGGING_TOKEN=$3 -GITHUB_USER="kedro-org" -GITHUB_REPO="kedro" - -# Array of GitHub mergeable states that are valid to proceed with automatic PR merging. -# Adding "unstable" can be convenient, as it allows to auto-merge pull requests that -# pass just the required CI checks (whereas "clean" requires all CI checks to pass, -# regardless of whether they are required or not). More info: -# https://docs.github.com/en/graphql/reference/enums#mergestatestatus -# https://github.com/octokit/octokit.net/issues/1763 -VALID_MERGEABLE_STATES=("clean" "unstable") - -find_github_pr() { - # Find a PR from source to target branch - # Returns PR number if GitHub returned exactly one such PR - endpoint="https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/pulls?base=${TARGET_BRANCH}&head=${GITHUB_USER}:${SOURCE_BRANCH}&state=open" - response=$(curl --silent --header "Authorization: token ${GITHUB_TAGGING_TOKEN}" "${endpoint}") - num_open_prs=$(echo "$response" | tr '\r\n' ' ' | jq "length") - if [ "$num_open_prs" -eq 1 ]; then - echo "$response" | tr '\r\n' ' ' | jq ".[0].number" - fi -} - -check_pr_mergeable() { - # Check that the given PR is in a mergeable state - pr=$1 - endpoint="https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/pulls/${pr}" - response=$(curl --silent --header "Authorization: token ${GITHUB_TAGGING_TOKEN}" "${endpoint}") - mergeable=$(echo "${response}" | tr '\r\n' ' ' | jq ".mergeable // false") # default to false - echo "PR ${pr} mergeable: ${mergeable}" - mergeable_state=$(echo "${response}" | tr '\r\n' ' ' | jq --raw-output ".mergeable_state // \"unknown\"") - echo "PR ${pr} mergeable_state: ${mergeable_state}" - [ "${mergeable}" == true ] && [[ " ${VALID_MERGEABLE_STATES[@]} " =~ " ${mergeable_state} " ]] -} - -toggle_merge_commits() { - # Turns merge commits on or off for the repository - allow_merge_commit=$1 - endpoint="https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}" - payload="{\"allow_merge_commit\": ${allow_merge_commit}}" - status_code=$(curl -X PATCH \ - --silent \ - --header "Authorization: token ${GITHUB_TAGGING_TOKEN}" \ - --header "Content-Type: application/json" \ - --data "${payload}" \ - --output /dev/null \ - --write-out "%{http_code}\n" \ - "${endpoint}") - [ "${status_code}" -eq 200 ] -} - -delete_git_ref() { - # Delete a reference - git_ref=$1 - endpoint="https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/git/refs/${git_ref}" - status_code=$(curl -X DELETE \ - --silent \ - --header "Authorization: token ${GITHUB_TAGGING_TOKEN}" \ - --output /dev/null \ - --write-out "%{http_code}\n" \ - "${endpoint}") - [ "${status_code}" -eq 204 ] -} - -merge_pr() { - # Merge a given PR using merge commit - pr=$1 - toggle_merge_commits true - endpoint="https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/pulls/${pr}/merge" - payload='{"merge_method": "merge"}' - response=$(curl -X PUT \ - --silent \ - --header "Authorization: token ${GITHUB_TAGGING_TOKEN}" \ - --header "Content-Type: application/json" \ - --data "${payload}" \ - "${endpoint}") - toggle_merge_commits false - merged=$(echo "${response}" | tr '\r\n' ' ' | jq ".merged // false") # default to false - if [ "${merged}" == true ]; then - echo "PR ${pr} successfully merged" - delete_git_ref "heads/${SOURCE_BRANCH}" - echo "Branch ${SOURCE_BRANCH} successfully deleted" - else - message=$(echo "${response}" | tr '\r\n' ' ' | jq --raw-output ".message") - echo "PR ${pr} NOT merged. Message: ${message}" - fi - [ "${merged}" == true ] -} - -pr_number=$(find_github_pr) - -if [ -z "${pr_number}" ]; then - echo "No PR found from ${SOURCE_BRANCH} to ${TARGET_BRANCH}" - exit 0 -fi - -if check_pr_mergeable "${pr_number}"; then - merge_pr "${pr_number}" -else - echo "PR ${pr_number} is not in a mergeable state" -fi diff --git a/tools/circleci/requirements.txt b/tools/circleci/requirements.txt deleted file mode 100644 index 4317e05e69..0000000000 --- a/tools/circleci/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pip>=21.2 -twine~=3.0 diff --git a/tools/circleci/rtd-build.sh b/tools/circleci/rtd-build.sh deleted file mode 100755 index aeecf2ece5..0000000000 --- a/tools/circleci/rtd-build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# Script to trigger the documentation build for Kedro in ReadTheDocs. - -# Exit script if you try to use an uninitialized variable. -set -o nounset - -# Exit script if a statement returns a non-true return value. -set -o errexit - -RTD_TOKEN=$1 -BUILD_VERSION=$2 # version of the docs to be built -RTD_ENDPOINT="https://readthedocs.org/api/v3/projects/kedro/versions/${BUILD_VERSION}/builds/" - -curl -X POST \ - --silent --show-error --fail --retry 3 \ - --header "Authorization: token ${RTD_TOKEN}" \ - --header "Content-Length: 0" \ - "${RTD_ENDPOINT}" diff --git a/tools/circleci/github_scripts/merge.sh b/tools/github_actions/merge.sh similarity index 97% rename from tools/circleci/github_scripts/merge.sh rename to tools/github_actions/merge.sh index d863061983..f1d6a036ac 100755 --- a/tools/circleci/github_scripts/merge.sh +++ b/tools/github_actions/merge.sh @@ -28,7 +28,7 @@ PAYLOAD=$(cat <<-END "title": "[AUTO-MERGE] Merge ${SOURCE_BRANCH} into ${TARGET_BRANCH} via ${PR_BRANCH}", "head": "${PR_BRANCH}", "base": "${TARGET_BRANCH}", - "body": "A new change in ${SOURCE_BRANCH} cannot be merged into ${TARGET_BRANCH} as part of the regular sync job, hence this PR. Please resolve the conflicts manually, and make sure to obtain 2 approvals once the builds pass.\\n\\n### IMPORTANT NOTICE\\n\\nPlease let CircleCI merge this PR automatically, with merge commit enabled." + "body": "A new change in ${SOURCE_BRANCH} cannot be merged into ${TARGET_BRANCH} as part of the regular sync job, hence this PR. Please resolve the conflicts manually, and make sure to obtain 2 approvals once the builds pass.\\n\\n### IMPORTANT NOTICE\\n\\nPlease let GitHub Actions merge this PR automatically, with merge commit enabled." } END )