Merge branch 'main' into ks/refactor_scoring_endpoint

.github/workflows/automerge_plugin-only_prs.yml

@@ -40,8 +40,9 @@ jobs:
             echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}"
           fi
 
+
   isautomerge:
-    name: Check if PR is labeled with 'automerge' or 'automerge-web'
+    name: Set as 'automerge' if PR is labeled with 'automerge' or 'automerge-web'
     runs-on: ubuntu-latest
     if: |
       contains( github.event.pull_request.labels.*.name, 'automerge') ||
@@ -54,6 +55,7 @@ jobs:
         run: |
           echo "::set-output name=AUTOMERGE::True"
 
+
   travis_success:
     name: Check if Travis build is successful
     runs-on: ubuntu-latest
@@ -66,7 +68,7 @@ jobs:
         id: gettravisstatus
         run: |
           echo ${{ github.event.pull_request.head.sha }}
-          echo "TRAVIS_CONCLUSION=$(curl -X GET https://api.github.com/repos/brain-score/language/commits/${{ github.event.pull_request.head.sha }}/check-runs | python -c "from __future__ import print_function; import sys,json; print(next(run['conclusion'] for run in json.load(sys.stdin)['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV
+          echo "TRAVIS_CONCLUSION=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/language/commits/$github.event.pull_request.head.sha/check-runs\"); print(next(run['conclusion'] for run in r.json()['check_runs'] if run['name'] == 'Travis CI - Pull Request'))")" >> $GITHUB_ENV
       - name: Check if Travis was successful
         id: istravisok
         run: |
@@ -81,89 +83,35 @@ jobs:
           fi
           echo "::set-output name=TRAVIS_OK::$travisok"
 
-  email_on_failure:
-    name: If tests fail on an automerge-web PR, send email to submitter
-    runs-on: ubuntu-latest
-    needs: [setup, travis_success]
-    env: 
-      BSC_DATABASESECRET: ${{ needs.setup.outputs.BSC_DATABASESECRET }}
-    if: |
-        needs.travis_success.outputs.TRAVIS_OK == 'False' &&
-        join(github.event.pull_request.labels.*.name, ',') == 'automerge-web'
-    steps:
-    - name: Parse Brain-Score user ID from PR title
-      run: |
-        echo "BS_UID="$(<<<${{ github.event.pull_request.title }} | sed -E 's/.*\(user:([^)]+)\).*/\1/'"" >> $GITHUB_ENV
-    - name: Build project
-      run: |
-        python -m pip install "."
-    - name: Get email from Brain-Score UID
-      run: |
-        echo "TO_EMAIL=$(python .github/workflows/call_endpoints.py get_user_email '${{ env.BS_UID }}')" >> $GITHUB_ENV
-    - name: Send email to Brain-Score user
-      uses: dawidd6/action-send-mail@v3
-      with:
-        # Required mail server address:
-        server_address: smtp.gmail.com
-        # Server port, default 25:
-        server_port: 465
-        # Optional whether this connection use TLS (default is true if server_port is 465)
-        secure: true
-        # Optional (recommended) mail server username:
-        username: ${{ secrets.MAIL_USERNAME }}
-        # Optional (recommended) mail server password:
-        password: ${{ secrets.MAIL_PASSWORD }}
-        # Required mail subject:
-        subject: Brain-Score submission failed
-        # Required recipients' addresses:
-        to: ${{ env.BS_UID }}
-        # Required sender full name:
-        from: Brain-Score
-        # Optional plain body:
-        body: Your Brain-Score submission did not pass checks. Please review the test results and update the PR at https://github.com/brain-score/language/pull/${{ github.event.number }} or send in an updated submission via the website.
-
-  checkchanges:
-    name: Check if PR only changes plugin files
+
+  plugin_only:
+    name: Ensure PR ONLY changes plugin files
     runs-on: ubuntu-latest
     needs: travis_success
     if: ${{ needs.travis_success.outputs.TRAVIS_OK == 'True' }}
     outputs:
-      PLUGIN_INFO: ${{ steps.getpluginfo.outputs.PLUGIN_INFO }}
-      IS_PLUGIN_ONLY: ${{ steps.ispluginonly.outputs.IS_PLUGIN_ONLY }}
+      PLUGIN_ONLY: ${{ steps.ispluginonly.outputs.PLUGIN_ONLY }}
     steps:
-      - name: Check out repository code
-        uses: actions/checkout@v2
-
-      - name: Get changed files
-        uses: dorny/[email protected]
-        id: filter
-        with:
-          list-files: shell
-          filters: |
-            changed:
-              - '**'
-      - name: Save changed files to env var
-        run: echo "CHANGED_FILES=${{ steps.filter.outputs.changed_files }}" >> $GITHUB_ENV
-
-      - name: Parse changed files with python script
-        run: |
-          echo "PLUGIN_INFO=$(python .github/workflows/parse_changed_files.py '${{ env.CHANGED_FILES }}')" >> $GITHUB_ENV
-          
-      - name: Save plugin info to outputs
-        id: getpluginfo
-        run: |
-          echo "PLUGIN_INFO=$PLUGIN_INFO" >> $GITHUB_OUTPUT
-
-      - name: check if plugin-only
+      - name: Parse plugin_only confirmation from Travis status update
+        id: getpluginonlyvalue
+        run: echo "PLUGIN_ONLY=$(python -c "import requests; r = requests.get(\"https://api.github.com/repos/brain-score/language/statuses/$github.event.pull_request.head.sha\"); print(next(status['description'].split('- ')[1] for status in r.json() if status['description'].startswith('Run automerge workflow')))")" >> $GITHUB_ENV
+      - name: Check if PR is plugin only
         id: ispluginonly
         run: |
-          echo "IS_PLUGIN_ONLY=$(jq -r '.is_plugin_only' <<< "$PLUGIN_INFO")" >> $GITHUB_OUTPUT
- 
+          if [ "$PLUGIN_ONLY" == "True" ]
+          then
+            pluginonly=True
+          else
+            pluginonly=False
+          fi
+          echo "::set-output name=PLUGIN_ONLY::$pluginonly"
+
+
   automerge:
     name: If plugin-only, approve and merge
     runs-on: ubuntu-latest
-    needs: checkchanges
-    if: ${{ needs.checkchanges.outputs.IS_PLUGIN_ONLY == 'True' }}
+    needs: plugin_only
+    if: ${{ needs.plugin_only.outputs.PLUGIN_ONLY == 'True' }}
     steps:
       - name: Auto Approve
         uses: hmarr/[email protected]

.github/workflows/score_new_plugins.yml

@@ -35,6 +35,7 @@ jobs:
             echo "::set-output name=BSC_DATABASESECRET:: ${{ secrets.BSC_DATABASESECRET_PROD }}"
           fi
 
+
   changes_models_or_benchmarks:
     name: Check if PR makes changes to /models or /benchmarks
     runs-on: ubuntu-latest
@@ -44,32 +45,19 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@v2
-
-      - name: Get changed files
-        uses: dorny/[email protected]
-        id: filter
         with:
-          list-files: shell
-          filters: |
-            changed:
-              - '**'
-      - name: Save changed files to env var
-        run: echo "CHANGED_FILES=${{ steps.filter.outputs.changed_files }}" >> $GITHUB_ENV
-
-      - name: Parse changed files with python script
-        run: |
-          echo "PLUGIN_INFO=$(python .github/workflows/parse_changed_files.py '${{ env.CHANGED_FILES }}')" >> $GITHUB_ENV
-          
-      - name: Save plugin info to outputs
-        id: getpluginfo
-        run: |
-          echo "PLUGIN_INFO=$PLUGIN_INFO" >> $GITHUB_OUTPUT
+          fetch-depth: 0
 
-      - name: check if scoring needed
-        id: runscore
+      - name: Save changed files to env var
+        run: echo "CHANGED_FILES=$(git diff --name-only origin/main origin/$GITHUB_HEAD_REF | tr '\n' ' ')"  >> $GITHUB_ENV
+
+      - name: Get scoring info
         run: |
+          python -m pip install .
+          echo "PLUGIN_INFO=$(python -c "from brainscore_core.plugin_management.parse_plugin_changes import get_scoring_info; get_scoring_info('${{ env.CHANGED_FILES }}', 'brainscore_language')" >> $GITHUB_OUTPUT
           echo "RUN_SCORE=$(jq -r '.run_score' <<< "$PLUGIN_INFO")" >> $GITHUB_OUTPUT
 
+
   get_submitter_info:
     name: Get PR author email and (if web submission) Brain-Score user ID
     runs-on: ubuntu-latest
@@ -92,6 +80,7 @@ jobs:
         run: |
           echo "The Brain-Score user ID is ${{ steps.getuid.outputs.BS_UID }}"
           echo "PLUGIN_INFO="$(<<<$PLUGIN_INFO jq '. + {user_id: ${{ steps.getuid.outputs.UID }} }')"" >> $GITHUB_ENV
+
       - name: Get PR author email from GitHub username
         id: getemail
         uses: evvanErb/[email protected]
@@ -104,6 +93,7 @@ jobs:
           echo "The PR author email is ${{ steps.getemail.outputs.email }}"
           echo "PLUGIN_INFO="$(<<<$PLUGIN_INFO jq '. + {author_email: "'${{ steps.getemail.outputs.email }}'"}')"" >> $GITHUB_OUTPUT
 
+
   runscore:
     name: Score plugins
     runs-on: ubuntu-latest
@@ -125,4 +115,4 @@ jobs:
       - name: Build project and run scoring
         run: |
           python -m pip install "."
-          python .github/workflows/call_endpoints.py call_jenkins '${{ env.PLUGIN_INFO }}'
+          python -c "from brainscore_language.submission.endpoints import call_jenkins; call_jenkins('$env.PLUGIN_INFO')"

.github/workflows/travis_trigger.sh

@@ -2,9 +2,10 @@
 
 GH_WORKFLOW_TRIGGER=$1
 TRAVIS_PULL_REQUEST_SHA=$2
+PLUGIN_ONLY=$3
 
 curl -L -X POST \
 -H "Authorization: token $GH_WORKFLOW_TRIGGER" \
--d '{"state": "success", "description": "Run automerge workflow", 
+-d '{"state": "success", "description": "Run automerge workflow - '$PLUGIN_ONLY'", 
   "context": "continuous-integration/travis"}' \
   "https://api.github.com/repos/brain-score/language/statuses/$TRAVIS_PULL_REQUEST_SHA"

.readthedocs.yml

@@ -1,4 +1,10 @@
+version: 2
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+   python: "3.8"
+
 python:
-   version: 3.8
    install:
       - requirements: docs/requirements.txt