Skip to content

Commit

Permalink
move submission logic from github actions into travis/core
Browse files Browse the repository at this point in the history
  • Loading branch information
Katherine Fairchild committed Oct 11, 2023
1 parent a8068a7 commit 2cb59b7
Show file tree
Hide file tree
Showing 8 changed files with 52 additions and 200 deletions.
96 changes: 22 additions & 74 deletions .github/workflows/automerge_plugin-only_prs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ jobs:
echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}"
fi
isautomerge:
name: Set as 'automerge' if PR is labeled with 'automerge' or 'automerge-web'
runs-on: ubuntu-latest
Expand All @@ -54,6 +55,7 @@ jobs:
run: |
echo "::set-output name=AUTOMERGE::True"
travis_success:
name: Check if Travis build is successful
runs-on: ubuntu-latest
Expand All @@ -66,7 +68,7 @@ jobs:
id: gettravisstatus
run: |
echo ${{ github.event.pull_request.head.sha }}
echo "TRAVIS_CONCLUSION=$(curl -X GET https://api.github.com/repos/brain-score/language/commits/${{ github.event.pull_request.head.sha }}/check-runs | python -c "from __future__ import print_function; import sys,json; print(next(run['conclusion'] for run in json.load(sys.stdin)['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV
echo "TRAVIS_CONCLUSION=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/language/commits/$github.event.pull_request.head.sha/check-runs\"); print(next(run['conclusion'] for run in r.json()['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV
- name: Check if Travis was successful
id: istravisok
run: |
Expand All @@ -81,89 +83,35 @@ jobs:
fi
echo "::set-output name=TRAVIS_OK::$travisok"
email_on_failure:
name: If tests fail on an automerge-web PR, send email to submitter
runs-on: ubuntu-latest
needs: [setup, travis_success]
env:
BSC_DATABASESECRET: ${{ needs.setup.outputs.BSC_DATABASESECRET }}
if: |
needs.travis_success.outputs.TRAVIS_OK == 'False' &&
join(github.event.pull_request.labels.*.name, ',') == 'automerge-web'
steps:
- name: Parse Brain-Score user ID from PR title
run: |
echo "BS_UID="$(<<<${{ github.event.pull_request.title }} | sed -E 's/.*\(user:([^)]+)\).*/\1/'"" >> $GITHUB_ENV
- name: Build project
run: |
python -m pip install "."
- name: Get email from Brain-Score UID
run: |
echo "TO_EMAIL=$(python .github/workflows/call_endpoints.py get_user_email '${{ env.BS_UID }}')" >> $GITHUB_ENV
- name: Send email to Brain-Score user
uses: dawidd6/action-send-mail@v3
with:
# Required mail server address:
server_address: smtp.gmail.com
# Server port, default 25:
server_port: 465
# Optional whether this connection use TLS (default is true if server_port is 465)
secure: true
# Optional (recommended) mail server username:
username: ${{ secrets.MAIL_USERNAME }}
# Optional (recommended) mail server password:
password: ${{ secrets.MAIL_PASSWORD }}
# Required mail subject:
subject: Brain-Score submission failed
# Required recipients' addresses:
to: ${{ env.BS_UID }}
# Required sender full name:
from: Brain-Score
# Optional plain body:
body: Your Brain-Score submission did not pass checks. Please review the test results and update the PR at https://github.com/brain-score/language/pull/${{ github.event.number }} or send in an updated submission via the website.

checkchanges:
name: Check if PR only changes plugin files
plugin_only:
name: Ensure PR ONLY changes plugin files
runs-on: ubuntu-latest
needs: travis_success
if: ${{ needs.travis_success.outputs.TRAVIS_OK == 'True' }}
outputs:
PLUGIN_INFO: ${{ steps.getpluginfo.outputs.PLUGIN_INFO }}
IS_PLUGIN_ONLY: ${{ steps.ispluginonly.outputs.IS_PLUGIN_ONLY }}
PLUGIN_ONLY: ${{ steps.ispluginonly.outputs.PLUGIN_ONLY }}
steps:
- name: Check out repository code
uses: actions/checkout@v2

- name: Get changed files
uses: dorny/[email protected]
id: filter
with:
list-files: shell
filters: |
changed:
- '**'
- name: Save changed files to env var
run: echo "CHANGED_FILES=${{ steps.filter.outputs.changed_files }}" >> $GITHUB_ENV

- name: Parse changed files with python script
run: |
echo "PLUGIN_INFO=$(python .github/workflows/parse_changed_files.py '${{ env.CHANGED_FILES }}')" >> $GITHUB_ENV
- name: Save plugin info to outputs
id: getpluginfo
run: |
echo "PLUGIN_INFO=$PLUGIN_INFO" >> $GITHUB_OUTPUT
- name: check if plugin-only
- name: Parse plugin_only confirmation from Travis status update
id: getpluginonlyvalue
run: echo "PLUGIN_ONLY=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/language/statuses/$github.event.pull_request.head.sha\"); print(next(status['description'].split('- ')[1] for status in r.json() if status['description'].startswith('Run automerge workflow')))")" >> $GITHUB_ENV
- name: Check if PR is plugin only
id: ispluginonly
run: |
echo "IS_PLUGIN_ONLY=$(jq -r '.is_plugin_only' <<< "$PLUGIN_INFO")" >> $GITHUB_OUTPUT
if [ "$PLUGIN_ONLY" == "True" ]
then
pluginonly=True
else
pluginonly=False
fi
echo "::set-output name=PLUGIN_ONLY::$pluginonly"
automerge:
name: If plugin-only, approve and merge
runs-on: ubuntu-latest
needs: checkchanges
if: ${{ needs.checkchanges.outputs.IS_PLUGIN_ONLY == 'True' }}
needs: plugin_only
if: ${{ needs.plugin_only.outputs.PLUGIN_ONLY == 'True' }}
steps:
- name: Auto Approve
uses: hmarr/[email protected]
Expand Down
10 changes: 0 additions & 10 deletions .github/workflows/call_endpoints.py

This file was deleted.

94 changes: 0 additions & 94 deletions .github/workflows/parse_changed_files.py

This file was deleted.

20 changes: 8 additions & 12 deletions .github/workflows/score_new_plugins.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ jobs:
echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}"
fi
changes_models_or_benchmarks:
name: Check if PR makes changes to /models or /benchmarks
runs-on: ubuntu-latest
Expand All @@ -56,20 +57,13 @@ jobs:
- name: Save changed files to env var
run: echo "CHANGED_FILES=${{ steps.filter.outputs.changed_files }}" >> $GITHUB_ENV

- name: Parse changed files with python script
run: |
echo "PLUGIN_INFO=$(python .github/workflows/parse_changed_files.py '${{ env.CHANGED_FILES }}')" >> $GITHUB_ENV
- name: Save plugin info to outputs
id: getpluginfo
run: |
echo "PLUGIN_INFO=$PLUGIN_INFO" >> $GITHUB_OUTPUT
- name: check if scoring needed
id: runscore
- name: Get scoring info
run: |
python -m pip install .
echo "PLUGIN_INFO=$(python -c "from brainscore_core.plugin_management.parse_plugin_changes import get_scoring_info; get_scoring_info('${{ env.CHANGED_FILES }}', 'brainscore_language')" >> $GITHUB_ENV
echo "RUN_SCORE=$(jq -r '.run_score' <<< "$PLUGIN_INFO")" >> $GITHUB_OUTPUT
get_submitter_info:
name: Get PR author email and (if web submission) Brain-Score user ID
runs-on: ubuntu-latest
Expand All @@ -92,6 +86,7 @@ jobs:
run: |
echo "The Brain-Score user ID is ${{ steps.getuid.outputs.BS_UID }}"
echo "PLUGIN_INFO="$(<<<$PLUGIN_INFO jq '. + {user_id: ${{ steps.getuid.outputs.UID }} }')"" >> $GITHUB_ENV
- name: Get PR author email from GitHub username
id: getemail
uses: evvanErb/[email protected]
Expand All @@ -104,6 +99,7 @@ jobs:
echo "The PR author email is ${{ steps.getemail.outputs.email }}"
echo "PLUGIN_INFO="$(<<<$PLUGIN_INFO jq '. + {author_email: "'${{ steps.getemail.outputs.email }}'"}')"" >> $GITHUB_OUTPUT
runscore:
name: Score plugins
runs-on: ubuntu-latest
Expand All @@ -125,4 +121,4 @@ jobs:
- name: Build project and run scoring
run: |
python -m pip install "."
python .github/workflows/call_endpoints.py call_jenkins '${{ env.PLUGIN_INFO }}'
python -c "from brainscore_language.submission.endpoints import call_jenkins; call_jenkins('$env.PLUGIN_INFO')"
4 changes: 2 additions & 2 deletions .github/workflows/travis_trigger.sh
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@

GH_WORKFLOW_TRIGGER=$1
TRAVIS_PULL_REQUEST_SHA=$2
STATUS=$3
PLUGIN_ONLY=$3

curl -L -X POST \
-H "Authorization: token $GH_WORKFLOW_TRIGGER" \
-d '{"state": "'$STATUS'", "description": "Run automerge workflow",
-d '{"state": "success", "description": "Run automerge workflow - '$PLUGIN_ONLY'",
"context": "continuous-integration/travis"}' \
"https://api.github.com/repos/brain-score/language/statuses/$TRAVIS_PULL_REQUEST_SHA"
17 changes: 14 additions & 3 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ language: python
env:
global:
- PYTEST_SETTINGS="not requires_gpu and not memory_intense and not slow and not travis_slow"
- MODIFIES_PLUGIN="false"
- PLUGIN_ONLY="false"
- MODIFIES_PLUGIN="False"
- PLUGIN_ONLY="False"
- WEB_SUBMISSION="False"
install:
- python -m pip install -e ".[test]"
# install conda for plugin runner
Expand All @@ -30,6 +31,16 @@ script:
- if [ "$MODIFIES_PLUGIN" = "True" ]; then python -c "from brainscore_core.plugin_management.parse_plugin_changes import run_changed_plugin_tests; run_changed_plugin_tests(\"${CHANGED_FILES}\", 'brainscore_language')"; fi
- if [ "$PRIVATE_ACCESS" = 1 ] && [ -n "${GITHUB_TOKEN}" ] && [ "$PLUGIN_ONLY" = "False" ]; then pytest -m "private_access and $PYTEST_SETTINGS"; fi
- if [ "$PRIVATE_ACCESS" != 1 ] && [ "$PLUGIN_ONLY" = "False" ]; then pytest -m "not private_access and $PYTEST_SETTINGS" --ignore "tests/test_submission"; fi
after_failure:
# if web submission, notify submitter via email
- PR_TITLE=$(curl https://github.com/${TRAVIS_REPO_SLUG}/pull/${TRAVIS_PULL_REQUEST} 2> /dev/null | grep "title" | head -1)
- if [[ "$PR_TITLE" = "brain-score.org (user:"* ]]; then WEB_SUBMISSION="True"; fi
- |
if $WEB_SUBMISSION = "True"; then
BRAINSCORE_UID=$(<<<$PR_TITLE | sed -E 's/.*\(user:([^)]+)\).*/\1/') &&
python -c "from brainscore_language.submission.endpoints import send_email_to_submitter; send_email_to_submitter(\"${BRAINSCORE_UID}\", 'language', \"${$TRAVIS_PULL_REQUEST}\", \"${$GMAIL_USERNAME}\", \"${GMAIL_PASSWORD}\")";
fi
jobs:
include:
Expand All @@ -46,4 +57,4 @@ jobs:
- stage: "Automerge check"
install: skip
if: type = pull_request
script: if [ "$PLUGIN_ONLY" = "True" ]; then bash ${TRAVIS_BUILD_DIR}/.github/workflows/travis_trigger.sh $GH_WORKFLOW_TRIGGER $TRAVIS_PULL_REQUEST_SHA success; fi
script: if [ "$PLUGIN_ONLY" = "True" ]; then bash ${TRAVIS_BUILD_DIR}/.github/workflows/travis_trigger.sh $GH_WORKFLOW_TRIGGER $TRAVIS_PULL_REQUEST_SHA $PLUGIN_ONLY; fi
9 changes: 5 additions & 4 deletions brainscore_language/submission/endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from typing import List, Union, Dict

from brainscore_core import Score, Benchmark
from brainscore_core.submission import UserManager, RunScoringEndpoint, DomainPlugins
from brainscore_core.submission import UserManager, RunScoringEndpoint, DomainPlugins, send_user_email
from brainscore_language import load_model, load_benchmark, score
from brainscore_language.submission import config

Expand Down Expand Up @@ -40,9 +40,10 @@ def score(self, model_identifier: str, benchmark_identifier: str) -> Score:
run_scoring_endpoint = RunScoringEndpoint(language_plugins, db_secret=config.get_database_secret())


def get_user_email(uid: int) -> str:
""" Convenience method for GitHub Actions to get a user's email if their web-submitted PR fails. """
return get_email_from_uid(uid)
def send_email_to_submitter(uid: int, domain: str, pr_number: str,
mail_username: str, mail_password:str ) -> str:
""" Convenience method for Travis to send a user email if their web-submitted PR fails. """
return send_user_email(uid, domain, pr_number, mail_username, mail_password)


def create_user(domain: str, email: str) -> int:
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ requires-python = ">=3.7"
dependencies = [
"tqdm",
"numpy>=1.21",
"brainscore_core@git+https://github.com/brain-score/core.git@kvf/reduce_plugin_tests",
"brainscore_core@git+https://github.com/brain-score/core.git@kvf/submission_cleanup",
"fire",
"scikit-learn",
# model_helpers dependencies
Expand Down

0 comments on commit 2cb59b7

Please sign in to comment.