diff --git a/.github/workflows/fuzzy-ci.yml b/.github/workflows/fuzzy-ci.yml
index 19e6344c44..a85db27a50 100644
--- a/.github/workflows/fuzzy-ci.yml
+++ b/.github/workflows/fuzzy-ci.yml
@@ -33,7 +33,7 @@ env:
   GH_API_COMMENTS: ${{ github.event.pull_request.comments_url }}
   GH_API_LABELS: ${{ github.event.pull_request.issue_url }}/labels
   GH_API_ARTIFACTS: ${{ github.event.pull_request.base.repo.url }}/actions/artifacts
-  TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  TOKEN: ${{ secrets.MERLINOIS_PAT }}
 
   # URL short-hands
   ACTIONS_RUNS_ENDPOINT: ${{ github.event.repository.html_url }}/actions/runs
@@ -293,8 +293,6 @@ jobs:
     name: Evaluate diffs
     runs-on: ubuntu-22.04
     needs: diff
-    permissions:
-      pull-requests: write
     env:
       earlier_diff_was_approved: ${{ contains(github.event.pull_request.labels.*.name, 'fuzzy-diff-looks-good') }}
       current_diff_exists: ${{ needs.diff.outputs.diff_exits }}
@@ -345,6 +343,11 @@ jobs:
               Previous sha256: ${{ steps.approved_diff_info.outputs.hash }}\n\
               Current sha256: $current_diff_hash"
               status=$(curl -sL -w "%{http_code}" -o output.txt -X DELETE -H "Authorization: Bearer $TOKEN" "$GH_API_LABELS/$LABEL_NAME")
+              if [ "$status" -ne 200 ]; then
+                echo "Something went wrong trying to remove the $LABEL_NAME label. Please, remove it manually."
+                cat output.txt
+              fi
+              exit 2
             fi
           else
             if $current_diff_exists; then
@@ -366,8 +369,6 @@ jobs:
       github.event.action == 'labeled' &&
       github.event.label.name == 'fuzzy-diff-looks-good'
     runs-on: ubuntu-22.04
-    permissions:
-      pull-requests: write
     steps:
       - name: Retreive diff artifact meta-data
         id: diff_metadata
@@ -383,10 +384,13 @@ jobs:
         env:
           id: ${{ steps.diff_metadata.outputs.id }}
         run: |
-          # FIXME (?)
           if [ -z $id ]; then
             printf "You seem to have tried to approve a diff that doesn't exist yet.\nWait for the diff to have been generated and then try again."
             status=$(curl -sL -w "%{http_code}" -o output.txt -X DELETE -H "Authorization: Bearer $TOKEN" "$GH_API_LABELS/$LABEL_NAME")
+            if [ "$status" -ne 200 ]; then
+              echo "Something went wrong trying to remove the $LABEL_NAME label. Please, remove it manually."
+              cat output.txt
+            fi
             exit 1
           else
             echo "Diff has been approved."
@@ -397,7 +401,7 @@ jobs:
           id: ${{ steps.diff_metadata.outputs.id }}
         run: |
          # Doing this manually, since actions/download-artifact only works on the same workflow run on which the artifact was uploaded
-         curl -sSLO -H "Authorization: Bearer $TOKEN" "$GH_API_ARTIFACTS/$id/zip" -D headers.txt
+         curl -sSLO -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "$GH_API_ARTIFACTS/$id/zip" -D headers.txt
 
       - name: Unzip downloaded diff
         run: |