From 2cb59b758e31f216aeeb59f9d2cb4d61bd0e48c3 Mon Sep 17 00:00:00 2001 From: Katherine Fairchild Date: Wed, 11 Oct 2023 12:48:37 -0400 Subject: [PATCH] move submission logic from github actions into travis/core --- .../workflows/automerge_plugin-only_prs.yml | 96 +++++-------------- .github/workflows/call_endpoints.py | 10 -- .github/workflows/parse_changed_files.py | 94 ------------------ .github/workflows/score_new_plugins.yml | 20 ++-- .github/workflows/travis_trigger.sh | 4 +- .travis.yml | 17 +++- brainscore_language/submission/endpoints.py | 9 +- pyproject.toml | 2 +- 8 files changed, 52 insertions(+), 200 deletions(-) delete mode 100644 .github/workflows/call_endpoints.py delete mode 100644 .github/workflows/parse_changed_files.py mode change 100644 => 100755 .github/workflows/travis_trigger.sh diff --git a/.github/workflows/automerge_plugin-only_prs.yml b/.github/workflows/automerge_plugin-only_prs.yml index fffff348..1897be11 100644 --- a/.github/workflows/automerge_plugin-only_prs.yml +++ b/.github/workflows/automerge_plugin-only_prs.yml @@ -40,6 +40,7 @@ jobs: echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}" fi + isautomerge: name: Set as 'automerge' if PR is labeled with 'automerge' or 'automerge-web' runs-on: ubuntu-latest @@ -54,6 +55,7 @@ jobs: run: | echo "::set-output name=AUTOMERGE::True" + travis_success: name: Check if Travis build is successful runs-on: ubuntu-latest @@ -66,7 +68,7 @@ jobs: id: gettravisstatus run: | echo ${{ github.event.pull_request.head.sha }} - echo "TRAVIS_CONCLUSION=$(curl -X GET https://api.github.com/repos/brain-score/language/commits/${{ github.event.pull_request.head.sha }}/check-runs | python -c "from __future__ import print_function; import sys,json; print(next(run['conclusion'] for run in json.load(sys.stdin)['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV + echo "TRAVIS_CONCLUSION=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/language/commits/$github.event.pull_request.head.sha/check-runs\"); print(next(run['conclusion'] for run in r.json()['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV - name: Check if Travis was successful id: istravisok run: | @@ -81,89 +83,35 @@ jobs: fi echo "::set-output name=TRAVIS_OK::$travisok" - email_on_failure: - name: If tests fail on an automerge-web PR, send email to submitter - runs-on: ubuntu-latest - needs: [setup, travis_success] - env: - BSC_DATABASESECRET: ${{ needs.setup.outputs.BSC_DATABASESECRET }} - if: | - needs.travis_success.outputs.TRAVIS_OK == 'False' && - join(github.event.pull_request.labels.*.name, ',') == 'automerge-web' - steps: - - name: Parse Brain-Score user ID from PR title - run: | - echo "BS_UID="$(<<<${{ github.event.pull_request.title }} | sed -E 's/.*\(user:([^)]+)\).*/\1/'"" >> $GITHUB_ENV - - name: Build project - run: | - python -m pip install "." - - name: Get email from Brain-Score UID - run: | - echo "TO_EMAIL=$(python .github/workflows/call_endpoints.py get_user_email '${{ env.BS_UID }}')" >> $GITHUB_ENV - - name: Send email to Brain-Score user - uses: dawidd6/action-send-mail@v3 - with: - # Required mail server address: - server_address: smtp.gmail.com - # Server port, default 25: - server_port: 465 - # Optional whether this connection use TLS (default is true if server_port is 465) - secure: true - # Optional (recommended) mail server username: - username: ${{ secrets.MAIL_USERNAME }} - # Optional (recommended) mail server password: - password: ${{ secrets.MAIL_PASSWORD }} - # Required mail subject: - subject: Brain-Score submission failed - # Required recipients' addresses: - to: ${{ env.BS_UID }} - # Required sender full name: - from: Brain-Score - # Optional plain body: - body: Your Brain-Score submission did not pass checks. Please review the test results and update the PR at https://github.com/brain-score/language/pull/${{ github.event.number }} or send in an updated submission via the website. - - checkchanges: - name: Check if PR only changes plugin files + + plugin_only: + name: Ensure PR ONLY changes plugin files runs-on: ubuntu-latest needs: travis_success if: ${{ needs.travis_success.outputs.TRAVIS_OK == 'True' }} outputs: - PLUGIN_INFO: ${{ steps.getpluginfo.outputs.PLUGIN_INFO }} - IS_PLUGIN_ONLY: ${{ steps.ispluginonly.outputs.IS_PLUGIN_ONLY }} + PLUGIN_ONLY: ${{ steps.ispluginonly.outputs.PLUGIN_ONLY }} steps: - - name: Check out repository code - uses: actions/checkout@v2 - - - name: Get changed files - uses: dorny/paths-filter@v2.11.1 - id: filter - with: - list-files: shell - filters: | - changed: - - '**' - - name: Save changed files to env var - run: echo "CHANGED_FILES=${{ steps.filter.outputs.changed_files }}" >> $GITHUB_ENV - - - name: Parse changed files with python script - run: | - echo "PLUGIN_INFO=$(python .github/workflows/parse_changed_files.py '${{ env.CHANGED_FILES }}')" >> $GITHUB_ENV - - - name: Save plugin info to outputs - id: getpluginfo - run: | - echo "PLUGIN_INFO=$PLUGIN_INFO" >> $GITHUB_OUTPUT - - - name: check if plugin-only + - name: Parse plugin_only confirmation from Travis status update + id: getpluginonlyvalue + run: echo "PLUGIN_ONLY=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/language/statuses/$github.event.pull_request.head.sha\"); print(next(status['description'].split('- ')[1] for status in r.json() if status['description'].startswith('Run automerge workflow')))")" >> $GITHUB_ENV + - name: Check if PR is plugin only id: ispluginonly run: | - echo "IS_PLUGIN_ONLY=$(jq -r '.is_plugin_only' <<< "$PLUGIN_INFO")" >> $GITHUB_OUTPUT - + if [ "$PLUGIN_ONLY" == "True" ] + then + pluginonly=True + else + pluginonly=False + fi + echo "::set-output name=PLUGIN_ONLY::$pluginonly" + + automerge: name: If plugin-only, approve and merge runs-on: ubuntu-latest - needs: checkchanges - if: ${{ needs.checkchanges.outputs.IS_PLUGIN_ONLY == 'True' }} + needs: plugin_only + if: ${{ needs.plugin_only.outputs.PLUGIN_ONLY == 'True' }} steps: - name: Auto Approve uses: hmarr/auto-approve-action@v3.1.0 diff --git a/.github/workflows/call_endpoints.py b/.github/workflows/call_endpoints.py deleted file mode 100644 index 053971fe..00000000 --- a/.github/workflows/call_endpoints.py +++ /dev/null @@ -1,10 +0,0 @@ -import ast -import sys - -from brainscore_language.submission.endpoints import call_jenkins, get_email_from_uid - - -if __name__ == '__main__': - function = getattr(sys.modules[__name__], sys.argv[1]) - args = ast.literal_eval(sys.argv[2]) - function(args) \ No newline at end of file diff --git a/.github/workflows/parse_changed_files.py b/.github/workflows/parse_changed_files.py deleted file mode 100644 index 511b8ca7..00000000 --- a/.github/workflows/parse_changed_files.py +++ /dev/null @@ -1,94 +0,0 @@ -import re -import sys -from pathlib import Path -from typing import List, Tuple - -PLUGIN_DIRS = ['models', 'benchmarks', 'data', 'metrics'] - - -def get_changed_files(changed_files: str) -> Tuple[List[str], List[str]]: - changed_files_list = changed_files.split() - - plugin_files_changed = [] - non_plugin_files_changed = [] - - for f in changed_files_list: - if not any(plugin_dir in f for plugin_dir in PLUGIN_DIRS): - non_plugin_files_changed.append(f) - else: - plugin_files_changed.append(f) - - return plugin_files_changed, non_plugin_files_changed - - -def is_plugin_only(plugins_dict: dict, non_plugin_files_changed: List[str]): - """ - Stores `plugins_dict['is_plugin_only']` `"True"` or `"False"` - depending on whether there are any changed non-plugin files. - """ - plugin_only = len(non_plugin_files_changed) > 0 - plugins_dict["is_plugin_only"] = "False" if plugin_only else "True" - - -def _get_registered_plugins(plugin_type: str, plugin_dirs: List[str]) -> List[str]: - """ - Searches all `plugin_type` __init.py__ files for registered plugins. - Returns list of identifiers for each registered plugin. - """ - registered_plugins = [] - - plugin_type_dir = Path(f'brainscore_language/{plugin_type}') - - for plugin_dirname in plugin_dirs: - plugin_dirpath = plugin_type_dir / plugin_dirname - init_file = plugin_dirpath / "__init__.py" - with open(init_file) as f: - registry_name = plugin_type.strip( - 's') + '_registry' # remove plural and determine variable name, e.g. "models" -> "model_registry" - plugin_registrations = [line for line in f if f"{registry_name}[" - in line.replace('\"', '\'')] - for line in plugin_registrations: - result = re.search(f'{registry_name}\[.*\]', line) - identifier = result.group(0)[len(registry_name) + 2:-2] # remove brackets and quotes - registered_plugins.append(identifier) - - return registered_plugins - - -def plugins_to_score(plugins_dict, plugin_files_changed) -> str: - plugins_dict["run_score"] = "False" - - scoring_plugin_types = ("models", "benchmarks") - scoring_plugin_paths = tuple(f'brainscore_language/{plugin_type}/' for plugin_type in scoring_plugin_types) - model_and_benchmark_files = [fname for fname in plugin_files_changed if fname.startswith(scoring_plugin_paths)] - if len(model_and_benchmark_files) > 0: - plugins_dict["run_score"] = "True" - for plugin_type in scoring_plugin_types: - plugin_dirs = set( - [plugin_name_from_path(fname) for fname in model_and_benchmark_files if f'/{plugin_type}/' in fname]) - plugins_to_score = _get_registered_plugins(plugin_type, plugin_dirs) - plugins_dict[f'new_{plugin_type}'] = ' '.join(plugins_to_score) - - -def serialize_dict(plugins_dict: dict) -> str: - return str(plugins_dict).replace('\'', '\"') - - -def plugin_name_from_path(path_relative_to_library: str) -> str: - """ - Returns the name of the plugin from the given path. - E.g. `plugin_name_from_path("brainscore_vision/models/mymodel")` will return `"mymodel"`. - """ - return path_relative_to_library.split('/')[2] - - -if __name__ == '__main__': - changed_files = sys.argv[1] - plugin_files_changed, non_plugin_files_changed = get_changed_files(changed_files) - - plugins_dict = {} - is_plugin_only(plugins_dict, non_plugin_files_changed) - plugins_to_score(plugins_dict, plugin_files_changed) - plugins_dict = serialize_dict(plugins_dict) - - print(plugins_dict) diff --git a/.github/workflows/score_new_plugins.yml b/.github/workflows/score_new_plugins.yml index e17db32e..05c5dd5f 100644 --- a/.github/workflows/score_new_plugins.yml +++ b/.github/workflows/score_new_plugins.yml @@ -35,6 +35,7 @@ jobs: echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}" fi + changes_models_or_benchmarks: name: Check if PR makes changes to /models or /benchmarks runs-on: ubuntu-latest @@ -56,20 +57,13 @@ jobs: - name: Save changed files to env var run: echo "CHANGED_FILES=${{ steps.filter.outputs.changed_files }}" >> $GITHUB_ENV - - name: Parse changed files with python script - run: | - echo "PLUGIN_INFO=$(python .github/workflows/parse_changed_files.py '${{ env.CHANGED_FILES }}')" >> $GITHUB_ENV - - - name: Save plugin info to outputs - id: getpluginfo - run: | - echo "PLUGIN_INFO=$PLUGIN_INFO" >> $GITHUB_OUTPUT - - - name: check if scoring needed - id: runscore + - name: Get scoring info run: | + python -m pip install . + echo "PLUGIN_INFO=$(python -c "from brainscore_core.plugin_management.parse_plugin_changes import get_scoring_info; get_scoring_info('${{ env.CHANGED_FILES }}', 'brainscore_language')" >> $GITHUB_ENV echo "RUN_SCORE=$(jq -r '.run_score' <<< "$PLUGIN_INFO")" >> $GITHUB_OUTPUT + get_submitter_info: name: Get PR author email and (if web submission) Brain-Score user ID runs-on: ubuntu-latest @@ -92,6 +86,7 @@ jobs: run: | echo "The Brain-Score user ID is ${{ steps.getuid.outputs.BS_UID }}" echo "PLUGIN_INFO="$(<<<$PLUGIN_INFO jq '. + {user_id: ${{ steps.getuid.outputs.UID }} }')"" >> $GITHUB_ENV + - name: Get PR author email from GitHub username id: getemail uses: evvanErb/get-github-email-by-username-action@v1.25 @@ -104,6 +99,7 @@ jobs: echo "The PR author email is ${{ steps.getemail.outputs.email }}" echo "PLUGIN_INFO="$(<<<$PLUGIN_INFO jq '. + {author_email: "'${{ steps.getemail.outputs.email }}'"}')"" >> $GITHUB_OUTPUT + runscore: name: Score plugins runs-on: ubuntu-latest @@ -125,4 +121,4 @@ jobs: - name: Build project and run scoring run: | python -m pip install "." - python .github/workflows/call_endpoints.py call_jenkins '${{ env.PLUGIN_INFO }}' + python -c "from brainscore_language.submission.endpoints import call_jenkins; call_jenkins('$env.PLUGIN_INFO')" diff --git a/.github/workflows/travis_trigger.sh b/.github/workflows/travis_trigger.sh old mode 100644 new mode 100755 index 71535029..28258865 --- a/.github/workflows/travis_trigger.sh +++ b/.github/workflows/travis_trigger.sh @@ -2,10 +2,10 @@ GH_WORKFLOW_TRIGGER=$1 TRAVIS_PULL_REQUEST_SHA=$2 -STATUS=$3 +PLUGIN_ONLY=$3 curl -L -X POST \ -H "Authorization: token $GH_WORKFLOW_TRIGGER" \ --d '{"state": "'$STATUS'", "description": "Run automerge workflow", +-d '{"state": "success", "description": "Run automerge workflow - '$PLUGIN_ONLY'", "context": "continuous-integration/travis"}' \ "https://api.github.com/repos/brain-score/language/statuses/$TRAVIS_PULL_REQUEST_SHA" \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 260043b3..3002447e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,9 @@ language: python env: global: - PYTEST_SETTINGS="not requires_gpu and not memory_intense and not slow and not travis_slow" - - MODIFIES_PLUGIN="false" - - PLUGIN_ONLY="false" + - MODIFIES_PLUGIN="False" + - PLUGIN_ONLY="False" + - WEB_SUBMISSION="False" install: - python -m pip install -e ".[test]" # install conda for plugin runner @@ -30,6 +31,16 @@ script: - if [ "$MODIFIES_PLUGIN" = "True" ]; then python -c "from brainscore_core.plugin_management.parse_plugin_changes import run_changed_plugin_tests; run_changed_plugin_tests(\"${CHANGED_FILES}\", 'brainscore_language')"; fi - if [ "$PRIVATE_ACCESS" = 1 ] && [ -n "${GITHUB_TOKEN}" ] && [ "$PLUGIN_ONLY" = "False" ]; then pytest -m "private_access and $PYTEST_SETTINGS"; fi - if [ "$PRIVATE_ACCESS" != 1 ] && [ "$PLUGIN_ONLY" = "False" ]; then pytest -m "not private_access and $PYTEST_SETTINGS" --ignore "tests/test_submission"; fi +after_failure: + # if web submission, notify submitter via email + - PR_TITLE=$(curl https://github.com/${TRAVIS_REPO_SLUG}/pull/${TRAVIS_PULL_REQUEST} 2> /dev/null | grep "title" | head -1) + - if [[ "$PR_TITLE" = "brain-score.org (user:"* ]]; then WEB_SUBMISSION="True"; fi + - | + if $WEB_SUBMISSION = "True"; then + BRAINSCORE_UID=$(<<<$PR_TITLE | sed -E 's/.*\(user:([^)]+)\).*/\1/') && + python -c "from brainscore_language.submission.endpoints import send_email_to_submitter; send_email_to_submitter(\"${BRAINSCORE_UID}\", 'language', \"${$TRAVIS_PULL_REQUEST}\", \"${$GMAIL_USERNAME}\", \"${GMAIL_PASSWORD}\")"; + fi + jobs: include: @@ -46,4 +57,4 @@ jobs: - stage: "Automerge check" install: skip if: type = pull_request - script: if [ "$PLUGIN_ONLY" = "True" ]; then bash ${TRAVIS_BUILD_DIR}/.github/workflows/travis_trigger.sh $GH_WORKFLOW_TRIGGER $TRAVIS_PULL_REQUEST_SHA success; fi + script: if [ "$PLUGIN_ONLY" = "True" ]; then bash ${TRAVIS_BUILD_DIR}/.github/workflows/travis_trigger.sh $GH_WORKFLOW_TRIGGER $TRAVIS_PULL_REQUEST_SHA $PLUGIN_ONLY; fi diff --git a/brainscore_language/submission/endpoints.py b/brainscore_language/submission/endpoints.py index 82734bef..47d27f84 100644 --- a/brainscore_language/submission/endpoints.py +++ b/brainscore_language/submission/endpoints.py @@ -2,7 +2,7 @@ from typing import List, Union, Dict from brainscore_core import Score, Benchmark -from brainscore_core.submission import UserManager, RunScoringEndpoint, DomainPlugins +from brainscore_core.submission import UserManager, RunScoringEndpoint, DomainPlugins, send_user_email from brainscore_language import load_model, load_benchmark, score from brainscore_language.submission import config @@ -40,9 +40,10 @@ def score(self, model_identifier: str, benchmark_identifier: str) -> Score: run_scoring_endpoint = RunScoringEndpoint(language_plugins, db_secret=config.get_database_secret()) -def get_user_email(uid: int) -> str: - """ Convenience method for GitHub Actions to get a user's email if their web-submitted PR fails. """ - return get_email_from_uid(uid) +def send_email_to_submitter(uid: int, domain: str, pr_number: str, + mail_username: str, mail_password:str ) -> str: + """ Convenience method for Travis to send a user email if their web-submitted PR fails. """ + return send_user_email(uid, domain, pr_number, mail_username, mail_password) def create_user(domain: str, email: str) -> int: diff --git a/pyproject.toml b/pyproject.toml index b254a026..b6d6ed0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ requires-python = ">=3.7" dependencies = [ "tqdm", "numpy>=1.21", - "brainscore_core@git+https://github.com/brain-score/core.git@kvf/reduce_plugin_tests", + "brainscore_core@git+https://github.com/brain-score/core.git@kvf/submission_cleanup", "fire", "scikit-learn", # model_helpers dependencies