Clean up and centralize submission pipeline in core

.github/workflows/automerge_plugin-only_prs.yml

@@ -40,6 +40,7 @@ jobs:
             echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}"
           fi
 
+
   isautomerge:
     name: Set as 'automerge' if PR is labeled with 'automerge' or 'automerge-web'
     runs-on: ubuntu-latest
@@ -54,6 +55,7 @@ jobs:
         run: |
           echo "::set-output name=AUTOMERGE::True"
 
+
   travis_success:
     name: Check if Travis build is successful
     runs-on: ubuntu-latest
@@ -66,7 +68,7 @@ jobs:
         id: gettravisstatus
         run: |
           echo ${{ github.event.pull_request.head.sha }}
-          echo "TRAVIS_CONCLUSION=$(curl -X GET https://api.github.com/repos/brain-score/language/commits/${{ github.event.pull_request.head.sha }}/check-runs | python -c "from __future__ import print_function; import sys,json; print(next(run['conclusion'] for run in json.load(sys.stdin)['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV
+          echo "TRAVIS_CONCLUSION=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/language/commits/$github.event.pull_request.head.sha/check-runs\"); print(next(run['conclusion'] for run in r.json()['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV
       - name: Check if Travis was successful
         id: istravisok
         run: |
@@ -81,89 +83,35 @@ jobs:
           fi
           echo "::set-output name=TRAVIS_OK::$travisok"
 
-  email_on_failure:
-    name: If tests fail on an automerge-web PR, send email to submitter
-    runs-on: ubuntu-latest
-    needs: [setup, travis_success]
-    env: 
-      BSC_DATABASESECRET: ${{ needs.setup.outputs.BSC_DATABASESECRET }}
-    if: |
-        needs.travis_success.outputs.TRAVIS_OK == 'False' &&
-        join(github.event.pull_request.labels.*.name, ',') == 'automerge-web'
-    steps:
-    - name: Parse Brain-Score user ID from PR title
-      run: |
-        echo "BS_UID="$(<<<${{ github.event.pull_request.title }} | sed -E 's/.*\(user:([^)]+)\).*/\1/'"" >> $GITHUB_ENV
-    - name: Build project
-      run: |
-        python -m pip install "."
-    - name: Get email from Brain-Score UID
-      run: |
-        echo "TO_EMAIL=$(python .github/workflows/call_endpoints.py get_user_email '${{ env.BS_UID }}')" >> $GITHUB_ENV
-    - name: Send email to Brain-Score user
-      uses: dawidd6/action-send-mail@v3
-      with:
-        # Required mail server address:
-        server_address: smtp.gmail.com
-        # Server port, default 25:
-        server_port: 465
-        # Optional whether this connection use TLS (default is true if server_port is 465)
-        secure: true
-        # Optional (recommended) mail server username:
-        username: ${{ secrets.MAIL_USERNAME }}
-        # Optional (recommended) mail server password:
-        password: ${{ secrets.MAIL_PASSWORD }}
-        # Required mail subject:
-        subject: Brain-Score submission failed
-        # Required recipients' addresses:
-        to: ${{ env.BS_UID }}
-        # Required sender full name:
-        from: Brain-Score
-        # Optional plain body:
-        body: Your Brain-Score submission did not pass checks. Please review the test results and update the PR at https://github.com/brain-score/language/pull/${{ github.event.number }} or send in an updated submission via the website.
-
-  checkchanges:
-    name: Check if PR only changes plugin files
+
+  plugin_only:
+    name: Ensure PR ONLY changes plugin files
     runs-on: ubuntu-latest
     needs: travis_success
     if: ${{ needs.travis_success.outputs.TRAVIS_OK == 'True' }}
     outputs:
-      PLUGIN_INFO: ${{ steps.getpluginfo.outputs.PLUGIN_INFO }}
-      IS_PLUGIN_ONLY: ${{ steps.ispluginonly.outputs.IS_PLUGIN_ONLY }}
+      PLUGIN_ONLY: ${{ steps.ispluginonly.outputs.PLUGIN_ONLY }}
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v2
-
-      - name: Get changed files
-        uses: dorny/[email protected]
-        id: filter
-        with:
-          list-files: shell
-          filters: |
-            changed:
-              - '**'
-      - name: Save changed files to env var
-        run: echo "CHANGED_FILES=${{ steps.filter.outputs.changed_files }}" >> $GITHUB_ENV
-
-      - name: Parse changed files with python script
-        run: |
-          echo "PLUGIN_INFO=$(python .github/workflows/parse_changed_files.py '${{ env.CHANGED_FILES }}')" >> $GITHUB_ENV
-
-      - name: Save plugin info to outputs
-        id: getpluginfo
-        run: |
-          echo "PLUGIN_INFO=$PLUGIN_INFO" >> $GITHUB_OUTPUT
-
-      - name: check if plugin-only
+      - name: Parse plugin_only confirmation from Travis status update
+        id: getpluginonlyvalue
+        run: echo "PLUGIN_ONLY=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/language/statuses/$github.event.pull_request.head.sha\"); print(next(status['description'].split('- ')[1] for status in r.json() if status['description'].startswith('Run automerge workflow')))")" >> $GITHUB_ENV
+      - name: Check if PR is plugin only
         id: ispluginonly
         run: |
-          echo "IS_PLUGIN_ONLY=$(jq -r '.is_plugin_only' <<< "$PLUGIN_INFO")" >> $GITHUB_OUTPUT
-
+          if [ "$PLUGIN_ONLY" == "True" ]
+          then
+            pluginonly=True
+          else
+            pluginonly=False
+          fi
+          echo "::set-output name=PLUGIN_ONLY::$pluginonly"
+
+
   automerge:
     name: If plugin-only, approve and merge
     runs-on: ubuntu-latest
-    needs: checkchanges
-    if: ${{ needs.checkchanges.outputs.IS_PLUGIN_ONLY == 'True' }}
+    needs: plugin_only
+    if: ${{ needs.plugin_only.outputs.PLUGIN_ONLY == 'True' }}
     steps:
       - name: Auto Approve
         uses: hmarr/[email protected]

.github/workflows/score_new_plugins.yml

@@ -35,6 +35,7 @@ jobs:
             echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}"
           fi
 
+
   changes_models_or_benchmarks:
     name: Check if PR makes changes to /models or /benchmarks
     runs-on: ubuntu-latest
@@ -56,20 +57,13 @@ jobs:
       - name: Save changed files to env var
         run: echo "CHANGED_FILES=${{ steps.filter.outputs.changed_files }}" >> $GITHUB_ENV
 
-      - name: Parse changed files with python script
-        run: |
-          echo "PLUGIN_INFO=$(python .github/workflows/parse_changed_files.py '${{ env.CHANGED_FILES }}')" >> $GITHUB_ENV
-
-      - name: Save plugin info to outputs
-        id: getpluginfo
-        run: |
-          echo "PLUGIN_INFO=$PLUGIN_INFO" >> $GITHUB_OUTPUT
-
-      - name: check if scoring needed
-        id: runscore
+      - name: Get scoring info
         run: |
+          python -m pip install .
+          echo "PLUGIN_INFO=$(python -c "from brainscore_core.plugin_management.parse_plugin_changes import get_scoring_info; get_scoring_info('${{ env.CHANGED_FILES }}', 'brainscore_language')" >> $GITHUB_OUTPUT
           echo "RUN_SCORE=$(jq -r '.run_score' <<< "$PLUGIN_INFO")" >> $GITHUB_OUTPUT
 
+
   get_submitter_info:
     name: Get PR author email and (if web submission) Brain-Score user ID
     runs-on: ubuntu-latest
@@ -92,6 +86,7 @@ jobs:
         run: |
           echo "The Brain-Score user ID is ${{ steps.getuid.outputs.BS_UID }}"
           echo "PLUGIN_INFO="$(<<<$PLUGIN_INFO jq '. + {user_id: ${{ steps.getuid.outputs.UID }} }')"" >> $GITHUB_ENV
+
       - name: Get PR author email from GitHub username
         id: getemail
         uses: evvanErb/[email protected]
@@ -104,6 +99,7 @@ jobs:
           echo "The PR author email is ${{ steps.getemail.outputs.email }}"
           echo "PLUGIN_INFO="$(<<<$PLUGIN_INFO jq '. + {author_email: "'${{ steps.getemail.outputs.email }}'"}')"" >> $GITHUB_OUTPUT
 
+
   runscore:
     name: Score plugins
     runs-on: ubuntu-latest
@@ -125,4 +121,4 @@ jobs:
       - name: Build project and run scoring
         run: |
           python -m pip install "."
-          python .github/workflows/call_endpoints.py call_jenkins '${{ env.PLUGIN_INFO }}'
+          python -c "from brainscore_language.submission.endpoints import call_jenkins; call_jenkins('$env.PLUGIN_INFO')"

.github/workflows/travis_trigger.sh

@@ -2,10 +2,10 @@
 
 GH_WORKFLOW_TRIGGER=$1
 TRAVIS_PULL_REQUEST_SHA=$2
-STATUS=$3
+PLUGIN_ONLY=$3
 
 curl -L -X POST \
 -H "Authorization: token $GH_WORKFLOW_TRIGGER" \
--d '{"state": "'$STATUS'", "description": "Run automerge workflow", 
+-d '{"state": "success", "description": "Run automerge workflow - '$PLUGIN_ONLY'", 
   "context": "continuous-integration/travis"}' \
   "https://api.github.com/repos/brain-score/language/statuses/$TRAVIS_PULL_REQUEST_SHA"

.travis.yml

@@ -2,8 +2,9 @@ language: python
 env:
   global:
     - PYTEST_SETTINGS="not requires_gpu and not memory_intense and not slow and not travis_slow"
-    - MODIFIES_PLUGIN="false"
-    - PLUGIN_ONLY="false"
+    - MODIFIES_PLUGIN="False"
+    - PLUGIN_ONLY="False"
+    - WEB_SUBMISSION="False"
 install:
   - python -m pip install -e ".[test]"
   # install conda for plugin runner
@@ -30,6 +31,16 @@ script:
   - if [ "$MODIFIES_PLUGIN" = "True" ]; then python -c "from brainscore_core.plugin_management.parse_plugin_changes import run_changed_plugin_tests; run_changed_plugin_tests(\"${CHANGED_FILES}\", 'brainscore_language')"; fi
   - if [ "$PRIVATE_ACCESS" = 1 ] && [ -n "${GITHUB_TOKEN}" ] && [ "$PLUGIN_ONLY" = "False" ]; then pytest -m "private_access and $PYTEST_SETTINGS"; fi
   - if [ "$PRIVATE_ACCESS" != 1 ] && [ "$PLUGIN_ONLY" = "False" ]; then pytest -m "not private_access and $PYTEST_SETTINGS" --ignore "tests/test_submission"; fi
+after_failure:
+  # if web submission, notify submitter via email
+  - PR_TITLE=$(curl https://github.com/${TRAVIS_REPO_SLUG}/pull/${TRAVIS_PULL_REQUEST} 2> /dev/null | grep "title" | head -1)
+  - if [[ "$PR_TITLE" = "brain-score.org (user:"* ]]; then WEB_SUBMISSION="True"; fi
+  - |
+    if $WEB_SUBMISSION = "True"; then
+      BRAINSCORE_UID=$(<<<$PR_TITLE | sed -E 's/.*\(user:([^)]+)\).*/\1/') && 
+      python -c "from brainscore_language.submission.endpoints import send_email_to_submitter; send_email_to_submitter(\"${BRAINSCORE_UID}\", 'language', \"${$TRAVIS_PULL_REQUEST}\", \"${$GMAIL_USERNAME}\", \"${GMAIL_PASSWORD}\")"; 
+    fi
+
 
 jobs:
   include:
@@ -46,4 +57,4 @@ jobs:
     - stage: "Automerge check"
       install: skip
       if: type = pull_request
-      script: if [ "$PLUGIN_ONLY" = "True" ]; then bash ${TRAVIS_BUILD_DIR}/.github/workflows/travis_trigger.sh $GH_WORKFLOW_TRIGGER $TRAVIS_PULL_REQUEST_SHA success; fi
+      script: if [ "$PLUGIN_ONLY" = "True" ]; then bash ${TRAVIS_BUILD_DIR}/.github/workflows/travis_trigger.sh $GH_WORKFLOW_TRIGGER $TRAVIS_PULL_REQUEST_SHA $PLUGIN_ONLY; fi

brainscore_language/submission/endpoints.py

@@ -40,15 +40,18 @@ def score(self, model_identifier: str, benchmark_identifier: str) -> Score:
 run_scoring_endpoint = RunScoringEndpoint(language_plugins, db_secret=config.get_database_secret())
 
 
-def get_user_email(uid: int) -> str:
-    """ Convenience method for GitHub Actions to get a user's email if their web-submitted PR fails. """
-    return get_email_from_uid(uid)
+def send_email_to_submitter(uid: int, domain: str, pr_number: str, 
+                            mail_username: str, mail_password:str ) -> str:
+    """ Send submitter an email if their web-submitted PR fails. """
+    subject = "Brain-Score submission failed"
+    body = f"Your Brain-Score submission did not pass checks. Please review the test results and update the PR at https://github.com/brain-score/{domain}/pull/{pr_number} or send in an updated submission via the website."
+    return send_user_email(uid, body, mail_username, mail_password)
 
 
-def create_user(domain: str, email: str) -> int:
-    user_manager = UserManager(domain, email, db_secret=config.get_database_secret())
-    new_user_id = user_manager()
-    return new_user_id
+def get_user_id(email: str) -> int:
+    user_manager = UserManager(db_secret=config.get_database_secret())
+    user_id = user_manager.get_uid(email)
+    return user_id
 
 
 def _get_ids(args_dict: Dict[str, Union[str, List]], key: str) -> Union[List, str, None]:
@@ -110,7 +113,7 @@ def parse_args() -> argparse.Namespace:
     args_dict = vars(args)
 
     if 'user_id' not in args_dict or args_dict['user_id'] == None:
-        new_user_id = create_user(args_dict['domain'], args_dict['author_email'])
+        user_id = get_user_id(args_dict['author_email'])
         args_dict['user_id'] = new_user_id
 
     run_scoring(args_dict)

pyproject.toml

@@ -14,7 +14,7 @@ requires-python = ">=3.7"
 dependencies = [
     "tqdm",
     "numpy>=1.21",
-    "brainscore_core@git+https://github.com/brain-score/core.git@kvf/reduce_plugin_tests",
+    "brainscore_core@git+https://github.com/brain-score/core.git@kvf/submission_cleanup",
     "fire",
     "scikit-learn",
     # model_helpers dependencies