diff --git a/.github/workflows/ci-deb.yml b/.github/workflows/ci-deb.yml index 4ce3b9838b35..3845a3145dee 100644 --- a/.github/workflows/ci-deb.yml +++ b/.github/workflows/ci-deb.yml @@ -5,6 +5,7 @@ on: branches-ignore: - coverity_scan - run-fuzzer** + - debug-fuzzer-** schedule: - cron: '0 20 * * *' diff --git a/.github/workflows/ci-freebsd.yml b/.github/workflows/ci-freebsd.yml index 71b7c79fc21a..dfb97f2b0f4e 100644 --- a/.github/workflows/ci-freebsd.yml +++ b/.github/workflows/ci-freebsd.yml @@ -5,6 +5,7 @@ on: branches-ignore: - coverity_scan - run-fuzzer** + - debug-fuzzer-** pull_request: env: diff --git a/.github/workflows/ci-rpm.yml b/.github/workflows/ci-rpm.yml index 806409ff1fe9..1b8f48897f3a 100644 --- a/.github/workflows/ci-rpm.yml +++ b/.github/workflows/ci-rpm.yml @@ -5,6 +5,7 @@ on: branches-ignore: - coverity_scan - run-fuzzer** + - debug-fuzzer-** schedule: - cron: '0 20 * * *' diff --git a/.github/workflows/ci-scheduled-fuzzing.yml b/.github/workflows/ci-scheduled-fuzzing.yml index 80bf1fbf0058..78a0cd29df18 100644 --- a/.github/workflows/ci-scheduled-fuzzing.yml +++ b/.github/workflows/ci-scheduled-fuzzing.yml @@ -14,6 +14,24 @@ # - 'run-fuzzer-3600': Start fuzzing all protocols for one hour # - 'run-fuzzer-radius-7200': Start fuzzing RADIUS for two hours # +# Fuzzing failures (full log output including backtraces and reproducers) are +# uploaded as "artifacts" for the GitHub Actions run. +# +# The following script can be used to list fuzzer failures and download +# reproducers for local reproduction: +# +# scripts/build/fuzzer-fetch-artifacts +# +# If local reproduction does not recreate the failure then you may wish to +# attempt the reproduction within a GitHub Actions runner. To do this push to +# the following branch: +# +# - 'debug-fuzzer-' +# +# This will perform a fuzzer-enabled build and then instead of fuzzing will +# launch a tmate session that can be used to access the environment. Further +# details at the end of this file. +# name: Scheduled fuzzing @@ -21,6 +39,7 @@ on: push: branches: - 'run-fuzzer**' + - 'debug-fuzzer-**' schedule: - cron: '0 4 * * *' @@ -89,6 +108,9 @@ jobs: PROTOS=( "${PROTOS%-*}" ) elif [[ "$GITHUB_REF" = refs/heads/run-fuzzer-* ]]; then TOTAL_RUNTIME=${GITHUB_REF#refs/heads/run-fuzzer-} + elif [[ "$GITHUB_REF" = refs/heads/debug-fuzzer-* ]]; then + PROTOS=${GITHUB_REF#refs/heads/debug-fuzzer-} + PROTOS=( "${PROTOS%-*}" ) fi P=$( for i in ${!PROTOS[@]}; do @@ -162,6 +184,7 @@ jobs: key: corpus-${{ matrix.env.PROTOCOL }}-${{ steps.corpusparams.outputs.corpusct }}-${{ github.run_number }} restore-keys: | corpus-${{ matrix.env.PROTOCOL }}-${{ steps.corpusparams.outputs.corpusct }}- + if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }} - name: Package manager performance improvements run: | @@ -236,6 +259,7 @@ jobs: env: GITHUB_REF: "${{ github.ref }}" START_TIMESTAMP: "${{ needs.set-matrix.outputs.starttimestamp }}" + if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }} - name: "Clang libFuzzer: Store assets on failure" uses: actions/upload-artifact@v2 @@ -243,7 +267,7 @@ jobs: name: clang-fuzzer-${{ matrix.env.PROTOCOL }}-${{ steps.pick_commit.outputs.commit_id }} path: build/fuzzer retention-days: 30 - if: ${{ failure() }} + if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer') && failure() }} # # Merge the corpus which will be stored in the cache for the next run @@ -251,6 +275,7 @@ jobs: - name: Merge the corpus run: | make test.fuzzer.$PROTOCOL.merge + if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') }} # # We can push the LFS file directly, but we must use the GitHub API to @@ -273,4 +298,43 @@ jobs: fi env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - if: ${{ steps.corpusparams.outputs.corpusage > 2592000 && github.repository_owner == 'FreeRADIUS' }} + if: ${{ !startsWith(github.ref, 'refs/heads/debug-fuzzer-') && steps.corpusparams.outputs.corpusage > 2592000 && github.repository_owner == 'FreeRADIUS' }} + + + # + # If we are on the 'debug-fuzzer-*' branch then we start a tmate session to + # provide interactive shell access to the session so that the reproducers + # can be attempted in an identical environment to which the scheduled + # fuzzing occurred. + # + # The SSH rendezvous point will be emited continuously in the job output, + # which will look something like: + # + # SSH: ssh VfuX8SrNuU5pGPMyZcz7TpJTa@sfo2.tmate.io + # + # For example: + # + # git push origin debug-fuzzer-radius --force + # + # Look at the job output in: https://github.com/FreeRADIUS/freeradius-server/actions + # + # ssh VfuX8SrNuU5pGPMyZcz7TpJTa@sfo2.tmate.io + # + # Access requires that you have the private key corresponding to the + # public key of the GitHub user that initiated the job. + # + # Within this session you can use scripts/build/fuzzer-fetch-artifacts to + # download the reproducers just as you would do locally, e.g. + # + # export GITHUB_TOKEN= + # scripts/build/fuzzer-fetch-artifacts + # scripts/build/fuzzer-fetch-artifacts https://api.github.com/repos/FreeRADIUS/freeradius-server/actions/artifacts/186571481/zip + # scripts/build/fuzzer build/fuzzer/radius/crash-f1536d0fa2de775038e5dab74d233487a7cde819 + # + - name: "Debug: Start tmate" + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: ${{ startsWith(github.ref, 'refs/heads/debug-fuzzer-') && always() }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e6f90d43f52..4adf98b92111 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: branches-ignore: - coverity_scan - run-fuzzer** + - debug-fuzzer-** pull_request: env: diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 1efbb17a9e58..07bd06e1c47a 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -5,6 +5,7 @@ on: branches-ignore: - coverity_scan - run-fuzzer** + - debug-fuzzer-** pull_request: jobs: