diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 233c7fb51e5..1c6c65e278c 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -36,6 +36,8 @@ You can of course, as always, ask for help at [#coderbus](irc://irc.rizon.net/co You need the .NET 8.0 SDK, node>=v20, and npm>=v5.7 (in your PATH) to compile the server. On Linux, you also need the `libgdiplus` package installed to generate icons. +You need to run `corepack enable` to configure node to correctly build the webpanel. + The recommended IDE is Visual Studio 2022 or VSCode. In order to build the service version and/or the Windows installer you need a to run on Windows. diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 54007f37f3e..e70b298c560 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -25,90 +25,65 @@ on: branches: - dev - master - pull_request: - branches: - - dev - - master - pull_request_target: - types: [ opened, reopened, labeled, synchronize ] - branches: - - dev - - master + workflow_call: + inputs: + pull_request_number: + description: 'Pull Request Number' + required: true + type: string env: TGS_DOTNET_VERSION: 8 OD_MIN_COMPAT_DOTNET_VERSION: 7 OD_DOTNET_VERSION: 8 TGS_DOTNET_QUALITY: ga + TGS_WEBPANEL_NODE_VERSION: 20.x TGS_TEST_GITHUB_TOKEN: ${{ secrets.LIVE_TESTS_TOKEN }} TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} PACKAGING_PRIVATE_KEY_PASSPHRASE: ${{ secrets.PACKAGING_PRIVATE_KEY_PASSPHRASE }} - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} concurrency: - group: "ci-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" + group: "ci-${{ (github.event_name != 'push' && github.event_name != 'schedule' && github.event.inputs.pull_request_number) || github.run_id }}-${{ github.event_name }}" cancel-in-progress: true jobs: - security-checkpoint: - name: Check CI Clearance + build-releasenotes: + name: Build ReleaseNotes for Other Jobs runs-on: ubuntu-latest - permissions: - pull-requests: write - if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' steps: - - name: Comment on new Fork PR - if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 - uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 + - name: Setup dotnet + uses: actions/setup-dotnet@v4 with: - message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Please note that any changes to the workflow file will not be reflected in the run. + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Comment on dependabot PR - if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id == 49699333 - uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 - with: - message: Set the milestone to the next minor version, check for supply chain attacks, and then add the `CI Cleared` label to allow CI to run. + - name: Checkout (Branch) + uses: actions/checkout@v4 + if: github.event_name == 'push' || github.event_name == 'schedule' - - name: "Remove Stale 'CI Cleared' Label" - if: github.event.action == 'synchronize' || github.event.action == 'reopened' - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 + - name: Checkout (PR Merge) + uses: actions/checkout@v4 + if: github.event_name != 'push' && github.event_name != 'schedule' with: - labels: CI Cleared + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: "Remove 'CI Approval Required' Label" - if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && contains(github.event.pull_request.labels.*.name, 'CI Cleared')) - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 - with: - labels: CI Approval Required + - name: Build ReleaseNotes + run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: "Add 'CI Approval Required' Label" - if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) - uses: actions-ecosystem/action-add-labels@bd52874380e3909a1ac983768df6976535ece7f8 + - name: Store ReleaseNotes Binaries + uses: actions/upload-artifact@v4 with: - labels: CI Approval Required - github_token: ${{ github.token }} - - - name: Fail Clearance Check if PR has Unlabeled new Commits from User - if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) - run: exit 1 - - start-ci-run-gate: - name: CI Start Gate - needs: security-checkpoint - runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && (needs.security-checkpoint.result == 'success' || (needs.security-checkpoint.result == 'skipped' && (github.event_name == 'push' || github.event_name == 'schedule' || ((github.event.pull_request.head.repo.id == github.event.pull_request.base.repo.id && github.event.pull_request.user.id != 49699333) && github.event_name != 'pull_request_target'))))) - steps: - - name: GitHub Requires at Least One Step for a Job - run: exit 0 + name: release_notes_bins + path: ./release_notes_bins/ code-scanning: - name: Code Scanning - needs: start-ci-run-gate + name: Run CodeQL runs-on: ubuntu-latest permissions: security-events: write actions: read - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -118,19 +93,21 @@ jobs: - name: Checkout (Branch) uses: actions/checkout@v4 - if: github.event_name == 'push' || github.event_name == 'schedule' - - name: Checkout (PR Merge) - uses: actions/checkout@v4 + - name: Read Current SHA + id: get-pr-sha if: github.event_name != 'push' && github.event_name != 'schedule' - with: - ref: "refs/pull/${{ github.event.number }}/merge" + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: csharp + - name: Setup Telemetry Key File + run: echo "fake_telemetry_key" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true @@ -141,8 +118,6 @@ jobs: dmapi-build: name: Build DMAPI - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -200,7 +175,14 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Build DMAPI Test Project run: | @@ -224,8 +206,6 @@ jobs: opendream-build: name: Build DMAPI (OpenDream) - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -254,7 +234,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Checkout OpenDream run: | @@ -264,11 +244,6 @@ jobs: git checkout ${{ matrix.committish }} git submodule update --init --recursive - - name: Restore OpenDream - run: | - cd $HOME/OpenDream - dotnet restore - - name: Build OpenDream run: | cd $HOME/OpenDream/OpenDreamPackageTool @@ -287,8 +262,6 @@ jobs: efcore-version-match: name: Check Nuget Versions Match Tools runs-on: ubuntu-latest - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Checkout (Branch) uses: actions/checkout@v4 @@ -298,7 +271,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Retrieve dotnet-ef Tool Version id: dotnet-ef-tool @@ -344,9 +317,8 @@ jobs: pages-build: name: Build gh-pages + needs: build-releasenotes runs-on: ubuntu-latest - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -362,25 +334,25 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: gh-pages Clone run: git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Build Changelog (Incremental) run: | mv $HOME/tgsdox/changelog.yml ./ 2>/dev/null - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --generate-full-notes + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --generate-full-notes - name: Generate App Token run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV rm ${{ runner.temp }}/installation_secret.txt @@ -421,8 +393,8 @@ jobs: docker-build: name: Build Docker Image runs-on: ubuntu-latest - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: tgs_telemetry_key.txt steps: - name: Checkout (Branch) uses: actions/checkout@v4 @@ -432,15 +404,21 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build Docker Image - run: docker build . -f build/Dockerfile + run: docker build . -f build/Dockerfile --build-arg TGS_TELEMETRY_KEY_FILE=${{ env.TGS_TELEMETRY_KEY_FILE }} + + - name: Delete Telemetry Key File + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} linux-unit-tests: name: Linux Tests - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -448,6 +426,7 @@ jobs: env: TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }} + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt runs-on: ubuntu-latest steps: - name: Install x86 libc Dependencies @@ -462,6 +441,11 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -470,14 +454,21 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Enable Corepack + run: corepack enable - - name: Restore - run: dotnet restore + - name: Setup Telemetry Key File + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWindows + - name: Delete Telemetry Key File + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -496,8 +487,6 @@ jobs: windows-unit-tests: name: Windows Tests - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -505,6 +494,7 @@ jobs: env: TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }} + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt runs-on: windows-latest steps: - name: Setup dotnet @@ -513,6 +503,11 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -521,14 +516,23 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Restore - run: dotnet restore + - name: Enable Corepack + run: corepack enable + + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWix + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -545,25 +549,19 @@ jobs: name: windows-unit-test-coverage-${{ matrix.configuration }} path: ./TestResults/ - windows-integration-test: + windows-integration-tests: name: Windows Live Tests - needs: [dmapi-build, opendream-build] - if: (!(cancelled() || failure()) && needs.dmapi-build.result == 'success' && needs.opendream-build.result == 'success') + needs: [ dmapi-build, opendream-build ] strategy: fail-fast: false matrix: database-type: [ 'SqlServer', 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ] watchdog-type: [ 'Basic', 'Advanced' ] configuration: [ 'Debug', 'Release' ] + env: + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt runs-on: windows-latest steps: - - name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later - shell: powershell - if: ${{ matrix.database-type == 'SqlServer' }} - run: | - Write-Host "Checking" - sqlcmd -l 600 -S "(localdb)\MSSQLLocalDB" -Q "SELECT @@VERSION;" - - name: Setup dotnet uses: actions/setup-dotnet@v4 with: @@ -573,6 +571,18 @@ jobs: ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later + shell: powershell + if: ${{ matrix.database-type == 'SqlServer' }} + run: | + Write-Host "Checking" + sqlcmd -l 600 -S "(localdb)\MSSQLLocalDB" -Q "SELECT @@VERSION;" + + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Set TGS_TEST_DUMP_API_SPEC if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'SqlServer' }} run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV @@ -637,14 +647,23 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Enable Corepack + run: corepack enable - - name: Restore - run: dotnet restore + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build run: dotnet build -c ${{ matrix.configuration }} tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -738,7 +757,6 @@ jobs: linux-integration-tests: name: Linux Live Tests needs: [dmapi-build, opendream-build] - if: (!(cancelled() || failure()) && needs.dmapi-build.result == 'success' && needs.opendream-build.result == 'success') services: # We start all dbs here so we can just code the stuff once mssql: image: ${{ (matrix.database-type == 'SqlServer') && 'mcr.microsoft.com/mssql/server:2019-latest' || '' }} @@ -786,8 +804,19 @@ jobs: database-type: [ 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ] watchdog-type: [ 'Basic', 'Advanced' ] configuration: [ 'Debug', 'Release' ] + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt runs-on: ubuntu-latest steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + ${{ env.TGS_DOTNET_VERSION }}.0.x + ${{ env.OD_DOTNET_VERSION }}.0.x + ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Disable ptrace_scope run: echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope @@ -797,14 +826,10 @@ jobs: sudo apt-get update sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb libgcc-s1:i386 libgdiplus - - name: Setup dotnet - uses: actions/setup-dotnet@v4 + - name: Setup Node.JS + uses: actions/setup-node@v4 with: - dotnet-version: | - ${{ env.TGS_DOTNET_VERSION }}.0.x - ${{ env.OD_DOTNET_VERSION }}.0.x - ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} - name: Set Sqlite Connection Info if: ${{ matrix.database-type == 'Sqlite' }} @@ -843,14 +868,21 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Enable Corepack + run: corepack enable - - name: Restore - run: dotnet restore + - name: Setup Telemetry Key File + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWindows tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj + - name: Delete Telemetry Key File + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -907,8 +939,7 @@ jobs: validate-openapi-spec: name: OpenAPI Spec Validation - needs: windows-integration-test - if: (!(cancelled() || failure()) && needs.windows-integration-test.result == 'success') + needs: windows-integration-tests runs-on: ubuntu-latest steps: - name: Install IBM OpenAPI Validator @@ -922,7 +953,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Retrieve OpenAPI Spec uses: actions/download-artifact@v4 @@ -935,8 +966,7 @@ jobs: upload-code-coverage: name: Upload Code Coverage - needs: [linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-test] - if: (!(cancelled() || failure()) && needs.linux-unit-tests.result == 'success' && needs.linux-integration-tests.result == 'success' && needs.windows-unit-tests.result == 'success' && needs.windows-integration-test.result == 'success') + needs: [ linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-tests, build-releasenotes ] runs-on: ubuntu-latest steps: - name: Checkout (Branch) @@ -947,7 +977,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Retrieve Linux Unit Test Coverage (Debug) uses: actions/download-artifact@v4 @@ -1183,23 +1213,35 @@ jobs: name: windows-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlite + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Upload Coverage to CodeCov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: directory: ./code_coverage fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + handle_no_reports_found: true + + - name: Wait for CodeCov Status + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes --wait-codecov ${{ github.run_id }} build-deb: name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595 - needs: start-ci-run-gate runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt steps: - name: Install Native Dependencies run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386 gnupg2 xmlstarlet libgdiplus + sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386 - name: Import GPG Key if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) @@ -1235,29 +1277,39 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Parse TGS version - run: | - echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + - name: Setup Telemetry Key File + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Execute Build Script (Unsigned) - if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev'))) + if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master')) run: sudo -E build/package/deb/build_package.sh - name: Execute Build Script (Signed) - if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) + if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master') env: PACKAGING_KEYGRIP: ${{ vars.PACKAGING_KEYGRIP }} + run: sudo -E build/package/deb/build_package.sh + + - name: Parse TGS version run: | - sudo -E build/package/deb/build_package.sh + echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV + + - name: Verify Package Files are Signed + if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master') + run: gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1.dsc gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1_amd64.changes gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1_amd64.buildinfo + - name: Delete Telemetry Key File + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Test Install run: | sudo mkdir /etc/tgstation-server @@ -1295,9 +1347,9 @@ jobs: build-msi: name: Build Windows Installer .exe - needs: start-ci-run-gate runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt steps: - name: Install winget uses: Cyberboss/install-winget@v1 @@ -1310,6 +1362,11 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -1318,7 +1375,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Restore Wix dotnet Tool run: | @@ -1328,12 +1385,21 @@ jobs: - name: Validate winget Manifest run: winget validate --manifest build/package/winget/manifest - - name: Restore - run: dotnet restore + - name: Enable Corepack + run: corepack enable + + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build Host run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj @@ -1426,8 +1492,7 @@ jobs: check-winget-pr-template: name: Check winget-pkgs Pull Request Template is up to date - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + needs: build-releasenotes runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -1436,12 +1501,6 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Retrieve Latest winget-pkgs PULL_REQUEST_TEMPLATE commit SHA from GitHub API - id: get-sha - run: | - curl -L -u "${{ vars.DEV_PUSH_USERNAME }}:${{ secrets.DEV_PUSH_TOKEN }}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" -o commits.json https://api.github.com/repos/microsoft/winget-pkgs/commits?path=.github/PULL_REQUEST_TEMPLATE.md - echo "pr_template_sha=$(cat commits.json | jq '.[0].sha')" >> $GITHUB_OUTPUT - - name: Checkout (Branch) uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -1450,58 +1509,42 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Restore - run: dotnet restore + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Retrieve Latest winget-pkgs PULL_REQUEST_TEMPLATE commit SHA from GitHub API + id: get-sha + run: | + curl -L -u "${{ vars.DEV_PUSH_USERNAME }}:${{ secrets.DEV_PUSH_TOKEN }}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" -o commits.json https://api.github.com/repos/microsoft/winget-pkgs/commits?path=.github/PULL_REQUEST_TEMPLATE.md + echo "pr_template_sha=$(cat commits.json | jq '.[0].sha')" >> $GITHUB_OUTPUT + + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins - name: Run ReleaseNotes Check - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --winget-template-check ${{ steps.get-sha.outputs.pr_template_sha }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --winget-template-check ${{ steps.get-sha.outputs.pr_template_sha }} - ci-completion-gate: # This job exists so there isn't a moving target for branch protections + ci-completion-gate: name: CI Completion Gate - needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match ] + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, efcore-version-match, code-scanning ] runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.code-scanning.result == 'success') steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Checkout (Branch) - uses: actions/checkout@v4 - if: github.event_name == 'push' || github.event_name == 'schedule' - - - name: Checkout (PR Merge) - uses: actions/checkout@v4 - if: github.event_name != 'push' && github.event_name != 'schedule' - with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - - name: Run ReleaseNotes Create CI Completion Check (PR HEAD) - if: github.event_name != 'push' && github.event_name != 'schedule' - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ci-completion-check ${{ github.event.pull_request.head.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} - - - name: Run ReleaseNotes Create CI Completion Check (Branch) - if: github.event_name == 'push' || github.event_name == 'schedule' - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ci-completion-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + - name: Mandatory Empty Step + run: exit 0 deployment-gate: name: Deployment Start Gate needs: ci-completion-gate runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.ci-completion-gate.result == 'success' && github.event_name == 'push' && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) + if: github.event_name == 'push' steps: - name: GitHub Requires at Least One Step for a Job run: exit 0 @@ -1510,7 +1553,7 @@ jobs: name: Deploy HTTP API needs: deployment-gate runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[APIDeploy]')) + if: contains(github.event.head_commit.message, '[APIDeploy]') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1521,12 +1564,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Parse API version shell: powershell run: | @@ -1546,13 +1583,19 @@ jobs: $ProgressPreference = 'SilentlyContinue' Invoke-WebRequest -Uri https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -OutFile changelog.yml + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_API_VERSION }} --httpapi + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll ${{ env.TGS_API_VERSION }} --httpapi - name: Generate App Token shell: powershell run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} $installSecret = Get-Content ${{ runner.temp }}/installation_secret.txt echo "INSTALLATION_TOKEN=$installSecret" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append rm ${{ runner.temp }}/installation_secret.txt @@ -1582,7 +1625,7 @@ jobs: name: Deploy DreamMaker API needs: deployment-gate runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[DMDeploy]')) + if: contains(github.event.head_commit.message, '[DMDeploy]') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1593,12 +1636,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Parse DMAPI version shell: powershell run: | @@ -1617,13 +1654,19 @@ jobs: $ProgressPreference = 'SilentlyContinue' Invoke-WebRequest -Uri https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -OutFile changelog.yml + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_DM_VERSION }} --dmapi + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll ${{ env.TGS_DM_VERSION }} --dmapi - name: Generate App Token shell: powershell run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} $installSecret = Get-Content ${{ runner.temp }}/installation_secret.txt echo "INSTALLATION_TOKEN=$installSecret" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append rm ${{ runner.temp }}/installation_secret.txt @@ -1653,7 +1696,7 @@ jobs: name: Deploy Nuget Packages needs: deployment-gate runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[NugetDeploy]')) + if: contains(github.event.head_commit.message, '[NugetDeploy]') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1664,17 +1707,17 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --nuget + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --nuget - name: Publish Tgstation.Server.Common to NuGet uses: alirezanet/publish-nuget@e276c40afeb2a154046f0997820f2a9ea74832d9 # v3.1.0 @@ -1704,7 +1747,7 @@ jobs: name: Ensure TGS Release is Latest GitHub Release needs: [deploy-dm, deploy-http] runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && (needs.deploy-dm.result == 'success' || needs.deploy-http.result == 'success') && !contains(github.event.head_commit.message, '[TGSDeploy]')) + if: (!contains(github.event.head_commit.message, '[TGSDeploy]')) steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1712,23 +1755,22 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Checkout - uses: actions/checkout@v4 - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins - name: Run ReleaseNotes with --ensure-release - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ensure-release ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ensure-release ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} deploy-tgs: name: Deploy TGS needs: [deploy-dm, deploy-http, deployment-gate] runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && github.event.ref == 'refs/heads/master' && contains(github.event.head_commit.message, '[TGSDeploy]')) + if: github.event.ref == 'refs/heads/master' && contains(github.event.head_commit.message, '[TGSDeploy]') + env: + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1736,28 +1778,37 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - name: Restore Wix dotnet Tool run: | cd build/package/winget dotnet tool restore -# We need to rebuild the installer.exe so it can be properly signed + - name: Enable Corepack + run: corepack enable - - name: Build Host + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + + - name: Build Host # We need to rebuild the installer.exe so it can be properly signed run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj - - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Prepare Artifacts shell: powershell run: build/package/winget/prepare_installer_input_artifacts.ps1 @@ -1842,13 +1893,19 @@ jobs: &"C:/Program Files/7-Zip/7z.exe" a ServerConsole.zip ./ServerConsole/* -tzip &"C:/Program Files/7-Zip/7z.exe" a ServerUpdatePackage.zip ./ServerUpdatePackage/* -tzip + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_VERSION }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll ${{ env.TGS_VERSION }} - name: Generate App Token shell: powershell run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} $installSecret = Get-Content ${{ runner.temp }}/installation_secret.txt echo "INSTALLATION_TOKEN=$installSecret" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append rm ${{ runner.temp }}/installation_secret.txt @@ -1948,7 +2005,6 @@ jobs: name: Regenerate Changelog runs-on: ubuntu-latest needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1956,26 +2012,23 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Checkout - uses: actions/checkout@v4 - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: gh-pages Clone run: git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Build Changelog (Incremental) run: | mv $HOME/tgsdox/changelog.yml ./ 2>/dev/null - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --generate-full-notes + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --generate-full-notes - name: Generate App Token run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV rm ${{ runner.temp }}/installation_secret.txt @@ -1998,7 +2051,6 @@ jobs: deploy-docker: name: Deploy TGS (Docker) needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') runs-on: ubuntu-latest steps: - name: Checkout @@ -2022,7 +2074,6 @@ jobs: deploy-ppa: name: Deploy TGS (PPA) needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') runs-on: ubuntu-latest steps: - name: Checkout @@ -2041,7 +2092,6 @@ jobs: deploy-winget: name: Deploy TGS (winget) needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') runs-on: windows-latest steps: - name: Setup dotnet @@ -2061,15 +2111,18 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes - - name: Retrieve Server Service uses: actions/download-artifact@v4 with: name: packaging-windows-raw-msi path: artifacts + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Execute Push Script shell: powershell run: build/package/winget/push_manifest.ps1 @@ -2082,4 +2135,4 @@ jobs: - name: Run ReleaseNotes with --link-winget shell: powershell - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --link-winget ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --link-winget ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml new file mode 100644 index 00000000000..d89d2d3055b --- /dev/null +++ b/.github/workflows/ci-security.yml @@ -0,0 +1,68 @@ +name: 'CI Security' + +on: + pull_request: + branches: + - dev + - master + pull_request_target: + types: [ opened, reopened, labeled, synchronize ] + branches: + - dev + - master + +concurrency: + group: "ci-security-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" + cancel-in-progress: true + +jobs: + security-checkpoint: + name: Check CI Clearance + if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Comment on new Fork PR + if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 + uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 + with: + message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Note that any changes to ci-security.yml will not be reflected in the run and the ci-pipeline.yml at the HEAD of the pull request will be used. + + - name: Comment on dependabot PR + if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id == 49699333 + uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 + with: + message: Set the milestone to the next minor version, check for supply chain attacks, and then add the `CI Cleared` label to allow CI to run. + + - name: "Remove Stale 'CI Cleared' Label" + if: github.event.action == 'synchronize' || github.event.action == 'reopened' + uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 + with: + labels: CI Cleared + + - name: "Remove 'CI Approval Required' Label" + if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && contains(github.event.pull_request.labels.*.name, 'CI Cleared')) + uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 + with: + labels: CI Approval Required + + - name: "Add 'CI Approval Required' Label" + if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) + uses: actions-ecosystem/action-add-labels@bd52874380e3909a1ac983768df6976535ece7f8 + with: + labels: CI Approval Required + github_token: ${{ github.token }} + + - name: Fail if PR has Unlabeled new Commits from User + if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) + run: exit 1 + + ci-pipline-workflow-call: + name: CI Pipeline + needs: security-checkpoint + if: (!(cancelled() || failure()) && (needs.security-checkpoint.result == 'success' || (github.event_name != 'pull_request_target' && github.event.pull_request.head.repo.id == github.event.pull_request.base.repo.id && github.event.pull_request.user.id != 49699333))) + uses: ./.github/workflows/ci-pipeline.yml + secrets: inherit + with: + pull_request_number: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/rerun-flaky-tests.yml b/.github/workflows/rerun-flaky-tests.yml index 9016637a498..806d89e3597 100644 --- a/.github/workflows/rerun-flaky-tests.yml +++ b/.github/workflows/rerun-flaky-tests.yml @@ -1,7 +1,7 @@ name: Rerun Flaky Live Tests on: workflow_run: - workflows: [CI Pipeline] + workflows: ['CI Pipeline', 'CI Security'] types: - completed jobs: diff --git a/.github/workflows/scripts/rerunFlakyTests.js b/.github/workflows/scripts/rerunFlakyTests.js index 5bacef12783..7dda0f389ad 100644 --- a/.github/workflows/scripts/rerunFlakyTests.js +++ b/.github/workflows/scripts/rerunFlakyTests.js @@ -3,7 +3,8 @@ const CONSIDERED_JOBS = [ "Windows Live Tests", "Linux Live Tests", - "Build .deb Package" + "Build .deb Package", + "Upload Code Coverage" ]; async function getFailedJobsForRun(github, context, workflowRunId, runAttempt) { @@ -31,17 +32,19 @@ export async function rerunFlakyTests({ github, context }) { context.payload.workflow_run.run_attempt ); - if (failingJobs.length > 1) { - console.log("Multiple jobs failing. PROBABLY not flaky, not rerunning."); + if (failingJobs.length > 3) { + console.log("Many jobs failing. PROBABLY not flaky, not rerunning."); return; } const filteredFailingJobs = failingJobs.filter((job) => { console.log(`Failing job: ${job.name}`) - return CONSIDERED_JOBS.some((title) => job.name.startsWith(title)); + return CONSIDERED_JOBS + .flatMap(jobName => [jobName, 'CI Pipeline / ' + jobName]) + .some((title) => job.name.startsWith(title)); }); - if (filteredFailingJobs.length === 0) { - console.log("Failing jobs are NOT designated flaky. Not rerunning."); + if (filteredFailingJobs.length !== failingJobs.length) { + console.log("One or more failing jobs are NOT designated flaky. Not rerunning."); return; } diff --git a/.github/workflows/size-label.yml b/.github/workflows/size-label.yml new file mode 100644 index 00000000000..096ce4c6432 --- /dev/null +++ b/.github/workflows/size-label.yml @@ -0,0 +1,16 @@ +name: Size Labelling +on: + pull_request_target: + +jobs: + size-label: + name: Add Size Label + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: size-label + uses: "pascalgn/size-label-action@bbbaa0d5ccce8e2e76254560df5c64b82dac2e12" # v0.5.2, consider upgrading after https://github.com/pascalgn/size-label-action/pull/54 is merged + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/stable-merge.yml b/.github/workflows/stable-merge.yml index 143a6198059..509c9e85c56 100644 --- a/.github/workflows/stable-merge.yml +++ b/.github/workflows/stable-merge.yml @@ -8,7 +8,7 @@ on: env: TGS_DOTNET_VERSION: 8 - OD_MIN_COMPAT_DOTNET_VERSION: 7 + TGS_DOTNET_QUALITY: ga jobs: master-merge: @@ -26,11 +26,6 @@ jobs: with: path: temp_workspace - - name: Restore - run: | - cd temp_workspace - dotnet restore - - name: Build ReleaseNotes run: | cd temp_workspace diff --git a/.github/workflows/update-ss13-org-mirror.yml b/.github/workflows/update-ss13-org-mirror.yml new file mode 100644 index 00000000000..f6562220f7b --- /dev/null +++ b/.github/workflows/update-ss13-org-mirror.yml @@ -0,0 +1,72 @@ +name: 'Sync spacestation13/tgstation-server' + +on: + push: + branches: + - dev + tags: + - '*' + workflow_dispatch: + +env: + TGS_DOTNET_VERSION: 8 + TGS_DOTNET_QUALITY: ga + +concurrency: + group: "ss13-mirror-sync" + cancel-in-progress: true + +jobs: + fork-sync: + name: Fork Sync + runs-on: ubuntu-latest + steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Build Checkout + uses: actions/checkout@v4 + with: + path: temp_workspace + + - name: Build ReleaseNotes + run: | + cd temp_workspace + dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Generate App Token + run: | + cd temp_workspace + dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} --spacestation13 + echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV + rm ${{ runner.temp }}/installation_secret.txt + env: + TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} + + - name: Main Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + token: ${{ env.INSTALLATION_TOKEN }} + + - name: Build ReleaseNotes + run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Generate App Token + run: | + dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV + rm ${{ runner.temp }}/installation_secret.txt + env: + TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} + + - name: Push to Spacestation13 Fork + run: | + git config user.name "tgstation-server-ci[bot]" + git config user.email "161980869+tgstation-server-ci[bot]@users.noreply.github.com" + git push "https://tgstation-server-ci:${{ env.INSTALLATION_TOKEN }}@github.com/spacestation13/tgstation-server" + git push --tags "https://tgstation-server-ci:${{ env.INSTALLATION_TOKEN }}@github.com/spacestation13/tgstation-server" diff --git a/README.md b/README.md index 794543dc854..a646eda7c05 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # tgstation-server -![CI Pipeline](https://github.com/tgstation/tgstation-server/workflows/CI%20Pipeline/badge.svg) [![codecov](https://codecov.io/gh/tgstation/tgstation-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tgstation/tgstation-server) +[![CI Pipeline](https://github.com/tgstation/tgstation-server/actions/workflows/ci-pipeline.yml/badge.svg)](https://github.com/tgstation/tgstation-server/actions/workflows/ci-pipeline.yml) [![codecov](https://codecov.io/gh/tgstation/tgstation-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tgstation/tgstation-server) [![GitHub license](https://img.shields.io/github/license/tgstation/tgstation-server.svg)](LICENSE) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/tgstation/tgstation-server.svg)](http://isitmaintained.com/project/tgstation/tgstation-server "Average time to resolve an issue") [![NuGet version](https://img.shields.io/nuget/v/Tgstation.Server.Api.svg)](https://www.nuget.org/packages/Tgstation.Server.Api) [![NuGet version](https://img.shields.io/nuget/v/Tgstation.Server.Client.svg)](https://www.nuget.org/packages/Tgstation.Server.Client) @@ -239,7 +239,7 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will - `General:InstanceLimit`: Maximum number of instances that may be created -- `General:GitHubAccessToken`: Specify a GitHub personal access token with no scopes here to highly mitigate the possiblity of 429 response codes from GitHub requests +- `General:GitHubAccessToken`: Specify a classic GitHub personal access token with no scopes here to highly mitigate the possiblity of 429 response codes from GitHub requests - `General:SkipAddingByondFirewallException`: Set to `true` if you have Windows firewall disabled @@ -313,6 +313,12 @@ The following providers use the `ServerUrl` setting: - Keycloak - InvisionCommunity +- `Telemetry:DisableVersionReporting`: Prevents you installation and the version you're using from being reported on the source repository's deployments list + +- `Telemetry:ServerFriendlyName`: Prevents anonymous TGS version usage statistics from being sent to be displayed on the repository. + +- `Telemetry:VersionReportingRepositoryId`: The repository telemetry is sent to. For security reasons, this is not the main TGS repo. See the [tgstation-server-deployments](https://github.com/tgstation/tgstation-server-deployments) repository for more information. + ### Database Configuration If using a MariaDB/MySQL server, our client library [recommends you set 'utf8mb4' as your default charset](https://github.com/PomeloFoundation/Pomelo.EntityFrameworkCore.MySql#1-recommended-server-charset) disregard at your own risk. diff --git a/build/Dockerfile b/build/Dockerfile index cba330156e2..b301591833e 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,5 +1,8 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim AS build +# Set in CI +ARG TGS_TELEMETRY_KEY_FILE= + # install node and npm # replace shell with bash so we can source files RUN curl --silent -o- https://raw.githubusercontent.com/creationix/nvm/v0.39.1/install.sh | sh @@ -17,7 +20,8 @@ RUN . $NVM_DIR/nvm.sh \ && apt-get install -y \ dos2unix \ libgdiplus \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && corepack enable # Build web control panel WORKDIR /repo/build @@ -45,7 +49,9 @@ RUN dotnet publish -c Release -o /app \ && build/RemoveUnsupportedRuntimes.sh /app WORKDIR /repo/src/Tgstation.Server.Host -RUN dotnet publish -c Release -o /app/lib/Default \ + +RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \ + && dotnet publish -c Release -o /app/lib/Default \ && cd ../.. \ && build/RemoveUnsupportedRuntimes.sh /app/lib/Default \ && mv /app/lib/Default/appsettings* /app diff --git a/build/TestCommon.props b/build/TestCommon.props index 1864c27e2a0..79449e89df3 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -18,9 +18,9 @@ - + - + diff --git a/build/Version.props b/build/Version.props index 0099619aac5..f524e2ee699 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,13 +3,13 @@ - 6.8.0 - 5.1.0 - 10.6.0 + 6.9.0 + 5.2.0 + 10.7.0 7.0.0 - 13.6.0 - 15.6.0 - 7.1.3 + 13.7.0 + 16.0.0 + 7.2.1 5.9.0 1.4.1 1.2.1 @@ -18,9 +18,8 @@ 8 https://download.visualstudio.microsoft.com/download/pr/751d3fcd-72db-4da2-b8d0-709c19442225/33cc492bde704bfd6d70a2b9109005a0/dotnet-hosting-8.0.6-win.exe - 10.11.8 + 11.4.2 - https://mirror.its.dal.ca/mariadb//mariadb-10.11.8/winx64-packages/mariadb-10.11.8-winx64.msi - 1.22.21 + diff --git a/build/WebpanelVersion.props b/build/WebpanelVersion.props index 806c5ed542d..2613f5510c0 100644 --- a/build/WebpanelVersion.props +++ b/build/WebpanelVersion.props @@ -1,6 +1,6 @@ - 5.9.0 + 6.1.0 diff --git a/build/package/deb/build_package.sh b/build/package/deb/build_package.sh index 1a25546da88..a48071f2193 100755 --- a/build/package/deb/build_package.sh +++ b/build/package/deb/build_package.sh @@ -18,7 +18,7 @@ apt-get install -y \ devscripts \ ca-certificates \ curl \ - gnupg \ + gnupg2 \ xmlstarlet \ libgdiplus @@ -35,6 +35,8 @@ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.co apt-get update apt-get install nodejs dotnet-sdk-8.0 -y +corepack enable + CURRENT_COMMIT=$(git rev-parse HEAD) rm -rf packaging @@ -67,7 +69,6 @@ cp build/tgstation-server.service debian/ SIGN_COMMAND="$SCRIPT_DIR/wrap_gpg.sh" rm -f /tmp/tgs_wrap_gpg_output.log - set +e if [[ -z "$PACKAGING_KEYGRIP" ]]; then diff --git a/build/package/deb/debian/control b/build/package/deb/debian/control index f0a7ecbbe3f..67356359b6e 100644 --- a/build/package/deb/debian/control +++ b/build/package/deb/debian/control @@ -24,5 +24,5 @@ Depends: Recommends: libsystemd0, gdb, -Description: A production scale tool for BYOND server management - This is a toolset to manage production BYOND servers. It includes the ability to update the server without having to stop or shutdown the server (the update will take effect on a "reboot" of the server), the ability to start the server and restart it if it crashes, as well as systems for managing code and game files, and locally merging GitHub Pull Requests for test deployments. +Description: A production scale tool for DreamMaker server management + This is a toolset to manage production DreamMaker servers. It includes the ability to update the server without having to stop or shutdown the server (the update will take effect on a "reboot" of the server), the ability to start the server and restart it if it crashes, as well as systems for managing code and game files, and locally merging GitHub Pull Requests for test deployments. diff --git a/build/package/deb/debian/rules b/build/package/deb/debian/rules index 7a13e96dd26..cbd846c2da9 100755 --- a/build/package/deb/debian/rules +++ b/build/package/deb/debian/rules @@ -10,7 +10,6 @@ override_dh_auto_clean: dotnet clean -c ReleaseNoWindows override_dh_auto_build: - dotnet restore cd src/Tgstation.Server.Host.Console && dotnet publish -c Release -o ../../artifacts cd src/Tgstation.Server.Host && dotnet publish -c Release -o ../../artifacts/lib/Default rm artifacts/lib/Default/appsettings.yml diff --git a/build/tgstation-server.service b/build/tgstation-server.service index 6f3c087584b..98c03fee57c 100644 --- a/build/tgstation-server.service +++ b/build/tgstation-server.service @@ -17,7 +17,7 @@ Restart=always KillMode=process ReloadSignal=SIGUSR2 RestartKillSignal=SIGUSR2 -AmbientCapabilities=CAP_SYS_NICE +AmbientCapabilities=CAP_SYS_NICE CAP_SYS_PTRACE WatchdogSec=60 WatchdogSignal=SIGTERM diff --git a/docs/Features.dox b/docs/Features.dox index 64eca5a3f7c..6600cb295f7 100644 --- a/docs/Features.dox +++ b/docs/Features.dox @@ -5,7 +5,7 @@ @section features_list Comprehensive Feature List -tgstation-server is a BYOND server managment suite. It includes all the following features +tgstation-server is a DreamMaker server managment suite. It includes all the following features - Standalone server with OpenAPI 3.0 defined HTTP REST API - Web based client included diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index 17464b44dae..4766b3dfe66 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,18 +1,19 @@ // tgstation-server DMAPI +// The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in IETF RFC 2119. -#define TGS_DMAPI_VERSION "7.1.3" +#define TGS_DMAPI_VERSION "7.2.1" // All functions and datums outside this document are subject to change with any version and should not be relied on. // CONFIGURATION -/// Create this define if you want to do TGS configuration outside of this file. +/// Consumers SHOULD create this define if you want to do TGS configuration outside of this file. #ifndef TGS_EXTERNAL_CONFIGURATION -// Comment this out once you've filled in the below. +// Consumers MUST comment this out once you've filled in the below and are not using [TGS_EXTERNAL_CONFIGURATION]. #error TGS API unconfigured -// Uncomment this if you wish to allow the game to interact with TGS 3.. +// Consumers MUST uncomment this if you wish to allow the game to interact with TGS version 3. // This will raise the minimum required security level of your game to TGS_SECURITY_TRUSTED due to it utilizing call()(). //#define TGS_V3_API @@ -52,7 +53,7 @@ #ifndef TGS_FILE2TEXT_NATIVE #ifdef file2text -#error Your codebase is re-defining the BYOND proc file2text. The DMAPI requires the native version to read the result of world.Export(). You can fix this by adding "#define TGS_FILE2TEXT_NATIVE file2text" before your override of file2text to allow the DMAPI to use the native version. This will only be used for world.Export(), not regular file accesses +#error Your codebase is re-defining the BYOND proc file2text. The DMAPI requires the native version to read the result of world.Export(). You SHOULD fix this by adding "#define TGS_FILE2TEXT_NATIVE file2text" before your override of file2text to allow the DMAPI to use the native version. This will only be used for world.Export(), not regular file accesses #endif #define TGS_FILE2TEXT_NATIVE file2text #endif @@ -152,16 +153,17 @@ //REQUIRED HOOKS /** - * Call this somewhere in [/world/proc/New] that is always run. This function may sleep! + * Consumers MUST call this somewhere in [/world/proc/New] that is always run. This function may sleep! * * * event_handler - Optional user defined [/datum/tgs_event_handler]. * * minimum_required_security_level: The minimum required security level to run the game in which the DMAPI is integrated. Can be one of [TGS_SECURITY_ULTRASAFE], [TGS_SECURITY_SAFE], or [TGS_SECURITY_TRUSTED]. + * * http_handler - Optional user defined [/datum/tgs_http_handler]. */ -/world/proc/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE) +/world/proc/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE, datum/tgs_http_handler/http_handler) return /** - * Call this when your initializations are complete and your game is ready to play before any player interactions happen. + * Consumers MUST call this when world initializations are complete and the game is ready to play before any player interactions happen. * * This may use [/world/var/sleep_offline] to make this happen so ensure no changes are made to it while this call is running. * Afterwards, consider explicitly setting it to what you want to avoid this BYOND bug: http://www.byond.com/forum/post/2575184 @@ -170,12 +172,10 @@ /world/proc/TgsInitializationComplete() return -/// Put this at the start of [/world/proc/Topic]. +/// Consumers MUST run this macro at the start of [/world/proc/Topic]. #define TGS_TOPIC var/tgs_topic_return = TgsTopic(args[1]); if(tgs_topic_return) return tgs_topic_return -/** - * Call this as late as possible in [world/proc/Reboot] (BEFORE ..()). - */ +/// Consumers MUST call this as late as possible in [world/proc/Reboot] (BEFORE ..()). /world/proc/TgsReboot() return @@ -269,7 +269,7 @@ /// The [/datum/tgs_chat_channel] the user was from. var/datum/tgs_chat_channel/channel -/// User definable handler for TGS events. +/// User definable handler for TGS events This abstract version SHOULD be overridden to be used. /datum/tgs_event_handler /// If the handler receieves [TGS_EVENT_HEALTH_CHECK] events. var/receive_health_checks = FALSE @@ -283,7 +283,41 @@ set waitfor = FALSE return -/// User definable chat command. +/// User definable handler for HTTP calls. This abstract version MUST be overridden to be used. +/datum/tgs_http_handler + +/** + * User definable callback for executing HTTP GET requests. + * MUST perform BYOND sleeps while the request is in flight. + * MUST return a [/datum/tgs_http_result]. + * SHOULD log its own errors + * + * url - The full URL to execute the GET request for including query parameters. + */ +/datum/tgs_http_handler/proc/PerformGet(url) + CRASH("[type]/PerformGet not implemented!") + +/// Result of a [/datum/tgs_http_handler] call. MUST NOT be overridden. +/datum/tgs_http_result + /// HTTP response as text + var/response_text + /// Boolean request success flag. Set for any 2XX response code. + var/success + +/** + * Create a [/datum/tgs_http_result]. + * + * * response_text - HTTP response as text. Must be provided in New(). + * * success - Boolean request success flag. Set for any 2XX response code. Must be provided in New(). + */ +/datum/tgs_http_result/New(response_text, success) + if(response_text && !istext(response_text)) + CRASH("response_text was not text!") + + src.response_text = response_text + src.success = success + +/// User definable chat command. This abstract version MUST be overridden to be used. /datum/tgs_chat_command /// The string to trigger this command on a chat bot. e.g `@bot name ...` or `!tgs name ...`. var/name = "" @@ -296,21 +330,27 @@ /** * Process command activation. Should return a [/datum/tgs_message_content] to respond to the issuer with. + * MUST be implemented * - * sender - The [/datum/tgs_chat_user] who issued the command. - * params - The trimmed string following the command `/datum/tgs_chat_command/var/name]. + * * sender - The [/datum/tgs_chat_user] who issued the command. + * * params - The trimmed string following the command `/datum/tgs_chat_command/var/name]. */ /datum/tgs_chat_command/proc/Run(datum/tgs_chat_user/sender, params) CRASH("[type] has no implementation for Run()") -/// User definable chat message. +/// User definable chat message. MUST NOT be overridden. /datum/tgs_message_content - /// The tring content of the message. Must be provided in New(). + /// The string content of the message. Must be provided in New(). var/text /// The [/datum/tgs_chat_embed] to embed in the message. Not supported on all chat providers. var/datum/tgs_chat_embed/structure/embed +/** + * Create a [/datum/tgs_message_content]. + * + * * text - The string content of the message. + */ /datum/tgs_message_content/New(text) ..() if(!istext(text)) @@ -319,7 +359,7 @@ src.text = text -/// User definable chat embed. Currently mirrors Discord chat embeds. See https://discord.com/developers/docs/resources/channel#embed-object-embed-structure for details. +/// User definable chat embed. Currently mirrors Discord chat embeds. See https://discord.com/developers/docs/resources/message#embed-object for details. /datum/tgs_chat_embed/structure var/title var/description @@ -331,13 +371,13 @@ /// Colour must be #AARRGGBB or #RRGGBB hex string. var/colour - /// See https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure for details. + /// See https://discord.com/developers/docs/resources/message#embed-object-embed-image-structure for details. var/datum/tgs_chat_embed/media/image - /// See https://discord.com/developers/docs/resources/channel#embed-object-embed-thumbnail-structure for details. + /// See https://discord.com/developers/docs/resources/message#embed-object-embed-thumbnail-structure for details. var/datum/tgs_chat_embed/media/thumbnail - /// See https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure for details. + /// See https://discord.com/developers/docs/resources/message#embed-object-embed-video-structure for details. var/datum/tgs_chat_embed/media/video var/datum/tgs_chat_embed/footer/footer @@ -346,7 +386,7 @@ var/list/datum/tgs_chat_embed/field/fields -/// Common datum for similar discord embed medias. +/// Common datum for similar Discord embed medias. /datum/tgs_chat_embed/media /// Must be set in New(). var/url @@ -354,6 +394,7 @@ var/height var/proxy_url +/// Create a [/datum/tgs_chat_embed]. /datum/tgs_chat_embed/media/New(url) ..() if(!istext(url)) @@ -361,13 +402,14 @@ src.url = url -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-footer-structure for details. +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-footer-structure for details. /datum/tgs_chat_embed/footer /// Must be set in New(). var/text var/icon_url var/proxy_icon_url +/// Create a [/datum/tgs_chat_embed/footer]. /datum/tgs_chat_embed/footer/New(text) ..() if(!istext(text)) @@ -375,16 +417,17 @@ src.text = text -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-provider-structure for details. +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-provider-structure for details. /datum/tgs_chat_embed/provider var/name var/url -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-author-structure for details. Must have name set in New(). +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-author-structure for details. Must have name set in New(). /datum/tgs_chat_embed/provider/author var/icon_url var/proxy_icon_url +/// Create a [/datum/tgs_chat_embed/footer]. /datum/tgs_chat_embed/provider/author/New(name) ..() if(!istext(name)) @@ -392,12 +435,15 @@ src.name = name -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-field-structure for details. Must have name and value set in New(). +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-field-structure for details. /datum/tgs_chat_embed/field + /// Must be set in New(). var/name + /// Must be set in New(). var/value var/is_inline +/// Create a [/datum/tgs_chat_embed/field]. /datum/tgs_chat_embed/field/New(name, value) ..() if(!istext(name)) diff --git a/src/DMAPI/tgs/core/README.md b/src/DMAPI/tgs/core/README.md index b82d8f49e29..965e21b549a 100644 --- a/src/DMAPI/tgs/core/README.md +++ b/src/DMAPI/tgs/core/README.md @@ -3,7 +3,7 @@ This folder contains all DMAPI code not directly involved in an API. - [_definitions.dm](./definitions.dm) contains defines needed across DMAPI internals. +- [byond_world_export.dm](./byond_world_export.dm) contains the default `/datum/tgs_http_handler` implementation which uses `world.Export()`. - [core.dm](./core.dm) contains the implementations of the `/world/proc/TgsXXX()` procs. Many map directly to the `/datum/tgs_api` functions. It also contains the /datum selection and setup code. - [datum.dm](./datum.dm) contains the `/datum/tgs_api` declarations that all APIs must implement. - [tgs_version.dm](./tgs_version.dm) contains the `/datum/tgs_version` definition -- diff --git a/src/DMAPI/tgs/core/byond_world_export.dm b/src/DMAPI/tgs/core/byond_world_export.dm new file mode 100644 index 00000000000..6ef8d841b8f --- /dev/null +++ b/src/DMAPI/tgs/core/byond_world_export.dm @@ -0,0 +1,22 @@ +/datum/tgs_http_handler/byond_world_export + +/datum/tgs_http_handler/byond_world_export/PerformGet(url) + // This is an infinite sleep until we get a response + var/export_response = world.Export(url) + TGS_DEBUG_LOG("byond_world_export: Export complete") + + if(!export_response) + TGS_ERROR_LOG("byond_world_export: Failed request: [url]") + return new /datum/tgs_http_result(null, FALSE) + + var/content = export_response["CONTENT"] + if(!content) + TGS_ERROR_LOG("byond_world_export: Failed request, missing content!") + return new /datum/tgs_http_result(null, FALSE) + + var/response_json = TGS_FILE2TEXT_NATIVE(content) + if(!response_json) + TGS_ERROR_LOG("byond_world_export: Failed request, failed to load content!") + return new /datum/tgs_http_result(null, FALSE) + + return new /datum/tgs_http_result(response_json, TRUE) diff --git a/src/DMAPI/tgs/core/core.dm b/src/DMAPI/tgs/core/core.dm index 15622228e91..63cb5a2c351 100644 --- a/src/DMAPI/tgs/core/core.dm +++ b/src/DMAPI/tgs/core/core.dm @@ -1,4 +1,4 @@ -/world/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE) +/world/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE, datum/tgs_http_handler/http_handler = null) var/current_api = TGS_READ_GLOBAL(tgs) if(current_api) TGS_ERROR_LOG("API datum already set (\ref[current_api] ([current_api]))! Was TgsNew() called more than once?") @@ -55,7 +55,10 @@ TGS_ERROR_LOG("Invalid parameter for event_handler: [event_handler]") event_handler = null - var/datum/tgs_api/new_api = new api_datum(event_handler, version) + if(!http_handler) + http_handler = new /datum/tgs_http_handler/byond_world_export + + var/datum/tgs_api/new_api = new api_datum(event_handler, version, http_handler) TGS_WRITE_GLOBAL(tgs, new_api) diff --git a/src/DMAPI/tgs/core/datum.dm b/src/DMAPI/tgs/core/datum.dm index f734fd0527f..3ca53e9bf7c 100644 --- a/src/DMAPI/tgs/core/datum.dm +++ b/src/DMAPI/tgs/core/datum.dm @@ -6,7 +6,7 @@ TGS_DEFINE_AND_SET_GLOBAL(tgs, null) var/list/warned_deprecated_command_runs -/datum/tgs_api/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version) +/datum/tgs_api/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version, datum/tgs_http_handler/http_handler) ..() src.event_handler = event_handler src.version = version diff --git a/src/DMAPI/tgs/includes.dm b/src/DMAPI/tgs/includes.dm index 23b714f9d06..f5118ed55a3 100644 --- a/src/DMAPI/tgs/includes.dm +++ b/src/DMAPI/tgs/includes.dm @@ -1,4 +1,5 @@ #include "core\_definitions.dm" +#include "core\byond_world_export.dm" #include "core\core.dm" #include "core\datum.dm" #include "core\tgs_version.dm" diff --git a/src/DMAPI/tgs/v5/api.dm b/src/DMAPI/tgs/v5/api.dm index 05d0dee25b3..3e328fc7c27 100644 --- a/src/DMAPI/tgs/v5/api.dm +++ b/src/DMAPI/tgs/v5/api.dm @@ -31,9 +31,12 @@ var/detached = FALSE -/datum/tgs_api/v5/New() + var/datum/tgs_http_handler/http_handler + +/datum/tgs_api/v5/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version, datum/tgs_http_handler/http_handler) . = ..() interop_version = version + src.http_handler = http_handler TGS_DEBUG_LOG("V5 API created: [json_encode(args)]") /datum/tgs_api/v5/ApiVersion() diff --git a/src/DMAPI/tgs/v5/bridge.dm b/src/DMAPI/tgs/v5/bridge.dm index 0c5e701a32b..62201fcc9e5 100644 --- a/src/DMAPI/tgs/v5/bridge.dm +++ b/src/DMAPI/tgs/v5/bridge.dm @@ -78,27 +78,24 @@ WaitForReattach(FALSE) TGS_DEBUG_LOG("Bridge request start") - // This is an infinite sleep until we get a response - var/export_response = world.Export(bridge_request) + var/datum/tgs_http_result/result = http_handler.PerformGet(bridge_request) TGS_DEBUG_LOG("Bridge request complete") - if(!export_response) - TGS_ERROR_LOG("Failed bridge request: [bridge_request]") + if(isnull(result)) + TGS_ERROR_LOG("Failed bridge request, handler returned null!") return - var/content = export_response["CONTENT"] - if(!content) - TGS_ERROR_LOG("Failed bridge request, missing content!") + if(!istype(result) || result.type != /datum/tgs_http_result) + TGS_ERROR_LOG("Failed bridge request, handler returned non-[/datum/tgs_http_result]!") return - var/response_json = TGS_FILE2TEXT_NATIVE(content) - if(!response_json) - TGS_ERROR_LOG("Failed bridge request, failed to load content!") + if(!result.success) + TGS_DEBUG_LOG("Failed bridge request, HTTP request failed!") return - var/list/bridge_response = json_decode(response_json) + var/list/bridge_response = json_decode(result.response_text) if(!bridge_response) - TGS_ERROR_LOG("Failed bridge request, bad json: [response_json]") + TGS_ERROR_LOG("Failed bridge request, bad json: [result.response_text]") return var/error = bridge_response[DMAPI5_RESPONSE_ERROR_MESSAGE] diff --git a/src/Tgstation.Server.Api/Models/JobCode.cs b/src/Tgstation.Server.Api/Models/JobCode.cs index be5db8d5a6b..1c20107b59f 100644 --- a/src/Tgstation.Server.Api/Models/JobCode.cs +++ b/src/Tgstation.Server.Api/Models/JobCode.cs @@ -108,5 +108,11 @@ public enum JobCode : byte /// [Description("Reconnect chat bot")] ReconnectChatBot, + + /// + /// When a repository is recloned. + /// + [Description("Reclone repository")] + RepositoryReclone, } } diff --git a/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs b/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs index 55d7f646e5c..2db5d354a75 100644 --- a/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs +++ b/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs @@ -16,5 +16,10 @@ public sealed class AdministrationResponse /// The latest available version of the Tgstation.Server.Host assembly from the upstream repository. If is not equal to 4 the update cannot be applied due to API changes. /// public Version? LatestVersion { get; set; } + + /// + /// This response is cached. This field indicates the when it was generated. + /// + public DateTimeOffset? GeneratedAt { get; set; } } } diff --git a/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs b/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs index cad5094d1a2..d0b86cdaa41 100644 --- a/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs +++ b/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs @@ -59,5 +59,11 @@ public sealed class DreamDaemonResponse : DreamDaemonApiBase /// [ResponseOptions] public bool? CurrentAllowWebclient { get; set; } + + /// + /// The amount of RAM in use by the game server in bytes. + /// + [ResponseOptions] + public long? ImmediateMemoryUsage { get; set; } } } diff --git a/src/Tgstation.Server.Api/Rights/RepositoryRights.cs b/src/Tgstation.Server.Api/Rights/RepositoryRights.cs index f9d134820bf..407437ce5d1 100644 --- a/src/Tgstation.Server.Api/Rights/RepositoryRights.cs +++ b/src/Tgstation.Server.Api/Rights/RepositoryRights.cs @@ -82,5 +82,10 @@ public enum RepositoryRights : ulong /// User may change submodule update settings. /// ChangeSubmoduleUpdate = 1 << 13, + + /// + /// User may trigger repository recloning. + /// + Reclone = 1 << 14, } } diff --git a/src/Tgstation.Server.Client/AdministrationClient.cs b/src/Tgstation.Server.Client/AdministrationClient.cs index 2d6841795cd..68c7fecd345 100644 --- a/src/Tgstation.Server.Client/AdministrationClient.cs +++ b/src/Tgstation.Server.Client/AdministrationClient.cs @@ -24,7 +24,7 @@ public AdministrationClient(IApiClient apiClient) } /// - public ValueTask Read(CancellationToken cancellationToken) => ApiClient.Read(Routes.Administration, cancellationToken); + public ValueTask Read(bool forceFresh, CancellationToken cancellationToken) => ApiClient.Read($"{Routes.Administration}?fresh={forceFresh}", cancellationToken); /// public async ValueTask Update( diff --git a/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs b/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs index cf4eb3ea874..1ea2dccb3b7 100644 --- a/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs +++ b/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs @@ -43,7 +43,10 @@ public DreamDaemonClient(IApiClient apiClient, Instance instance) public ValueTask Restart(CancellationToken cancellationToken) => apiClient.Patch(Routes.DreamDaemon, instance.Id!.Value, cancellationToken); /// - public ValueTask Read(CancellationToken cancellationToken) => apiClient.Read(Routes.DreamDaemon, instance.Id!.Value, cancellationToken); + public ValueTask Read(CancellationToken cancellationToken) => apiClient.Read( + Routes.DreamDaemon, + instance.Id!.Value, + cancellationToken); /// public ValueTask Update(DreamDaemonRequest dreamDaemon, CancellationToken cancellationToken) => apiClient.Update(Routes.DreamDaemon, dreamDaemon ?? throw new ArgumentNullException(nameof(dreamDaemon)), instance.Id!.Value, cancellationToken); diff --git a/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs b/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs index 7a7a6ead7cf..823c8d6a0ca 100644 --- a/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs +++ b/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs @@ -16,7 +16,7 @@ public interface IDreamDaemonClient /// /// The for the operation. /// A resulting in the information. - ValueTask Read(CancellationToken cancellationToken); + ValueTask Read(CancellationToken cancellationToken = default); /// /// Start . diff --git a/src/Tgstation.Server.Client/Components/IRepositoryClient.cs b/src/Tgstation.Server.Client/Components/IRepositoryClient.cs index 9d3bef806cf..51d8bdfd805 100644 --- a/src/Tgstation.Server.Client/Components/IRepositoryClient.cs +++ b/src/Tgstation.Server.Client/Components/IRepositoryClient.cs @@ -40,5 +40,12 @@ public interface IRepositoryClient /// The for the operation. /// A resulting in the . ValueTask Delete(CancellationToken cancellationToken); + + /// + /// Deletes and reclones the repository. + /// + /// The for the operation. + /// A resulting in the . + ValueTask Reclone(CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Client/Components/RepositoryClient.cs b/src/Tgstation.Server.Client/Components/RepositoryClient.cs index 124f1ec5b28..57116120247 100644 --- a/src/Tgstation.Server.Client/Components/RepositoryClient.cs +++ b/src/Tgstation.Server.Client/Components/RepositoryClient.cs @@ -44,5 +44,8 @@ public RepositoryClient(IApiClient apiClient, Instance instance) /// public ValueTask Update(RepositoryUpdateRequest repository, CancellationToken cancellationToken) => apiClient.Update(Routes.Repository, repository ?? throw new ArgumentNullException(nameof(repository)), instance.Id!.Value, cancellationToken); + + /// + public ValueTask Reclone(CancellationToken cancellationToken) => apiClient.Patch(Routes.Repository, instance.Id!.Value, cancellationToken); } } diff --git a/src/Tgstation.Server.Client/IAdministrationClient.cs b/src/Tgstation.Server.Client/IAdministrationClient.cs index 38d08983450..6d5d88f9510 100644 --- a/src/Tgstation.Server.Client/IAdministrationClient.cs +++ b/src/Tgstation.Server.Client/IAdministrationClient.cs @@ -17,9 +17,10 @@ public interface IAdministrationClient /// /// Get the represented by the . /// + /// If the response will be forcefully regenerated. /// The for the operation. /// A resulting in the represented by the . - ValueTask Read(CancellationToken cancellationToken); + ValueTask Read(bool forceFresh = false, CancellationToken cancellationToken = default); /// /// Updates the setttings. diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 89c048c68f2..3bca3608474 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -11,9 +11,9 @@ - + - + diff --git a/src/Tgstation.Server.Host/.config/dotnet-tools.json b/src/Tgstation.Server.Host/.config/dotnet-tools.json index af52642792d..a3847dcdfb2 100644 --- a/src/Tgstation.Server.Host/.config/dotnet-tools.json +++ b/src/Tgstation.Server.Host/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-ef": { - "version": "8.0.7", + "version": "8.0.8", "commands": [ "dotnet-ef" ] diff --git a/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs b/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs index 39ed128d591..1bd2a8b91e0 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs @@ -50,7 +50,7 @@ public RevisionCommand(IWatchdog watchdog, IRepositoryManager repositoryManager) public async ValueTask Invoke(string arguments, ChatUser user, CancellationToken cancellationToken) { string result; - if (arguments.Split(' ').Any(x => x.ToUpperInvariant() == "--REPO")) + if (arguments.Split(' ').Any(x => x.Equals("--repo", StringComparison.OrdinalIgnoreCase))) { if (repositoryManager.CloneInProgress || repositoryManager.InUse) return new MessageContent @@ -58,15 +58,13 @@ public async ValueTask Invoke(string arguments, ChatUser user, C Text = "Repository busy! Try again later", }; - using (var repo = await repositoryManager.LoadRepository(cancellationToken)) - { - if (repo == null) - return new MessageContent - { - Text = "Repository unavailable!", - }; - result = repo.Head; - } + using var repo = await repositoryManager.LoadRepository(cancellationToken); + if (repo == null) + return new MessageContent + { + Text = "Repository unavailable!", + }; + result = repo.Head; } else { diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs index 4e37e7d008b..9a04c0ac94e 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs @@ -79,13 +79,13 @@ await databaseContextFactory.UseContext( IGitHubService gitHubService; if (instanceAuthenticated) { - authenticatedGitHubService = gitHubServiceFactory.CreateService(repositorySettings.AccessToken!); + authenticatedGitHubService = await gitHubServiceFactory.CreateService(repositorySettings.AccessToken!, cancellationToken); gitHubService = authenticatedGitHubService; } else { authenticatedGitHubService = null; - gitHubService = gitHubServiceFactory.CreateService(); + gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); } var repoOwner = remoteInformation.RemoteRepositoryOwner!; @@ -175,8 +175,8 @@ public override async ValueTask> RemoveMergedTest } var gitHubService = repositorySettings.AccessToken != null - ? gitHubServiceFactory.CreateService(repositorySettings.AccessToken) - : gitHubServiceFactory.CreateService(); + ? await gitHubServiceFactory.CreateService(repositorySettings.AccessToken, cancellationToken) + : await gitHubServiceFactory.CreateService(cancellationToken); var tasks = revisionInformation .ActiveTestMerges @@ -255,7 +255,7 @@ protected override async ValueTask CommentOnTestMergeSource( int testMergeNumber, CancellationToken cancellationToken) { - var gitHubService = gitHubServiceFactory.CreateService(repositorySettings.AccessToken!); + var gitHubService = await gitHubServiceFactory.CreateService(repositorySettings.AccessToken!, cancellationToken); try { @@ -343,7 +343,7 @@ await databaseContextFactory.UseContext( return; } - var gitHubService = gitHubServiceFactory.CreateService(gitHubAccessToken); + var gitHubService = await gitHubServiceFactory.CreateService(gitHubAccessToken, cancellationToken); try { diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index c7c73e3a231..40fbde1a40c 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -146,7 +146,7 @@ await IOManager.DeleteDirectory( } /// - public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? progressReporter, CancellationToken cancellationToken) + public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter progressReporter, CancellationToken cancellationToken) { CheckVersionValidity(version); diff --git a/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs index 91887e73144..c24eee3e8b0 100644 --- a/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs @@ -37,7 +37,7 @@ public IEngineInstallation CreateInstallation(EngineVersion version, string path => DelegateCall(version, installer => installer.CreateInstallation(version, path, installationTask)); /// - public ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken) + public ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken) => DelegateCall(version, installer => installer.DownloadVersion(version, jobProgressReporter, cancellationToken)); /// diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs index 12cf8e657cd..6ca2b940306 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs @@ -52,7 +52,7 @@ protected EngineInstallerBase(IIOManager ioManager, ILogger public abstract ValueTask UpgradeInstallation(EngineVersion version, string path, CancellationToken cancellationToken); /// - public abstract ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken); + public abstract ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken); /// public abstract ValueTask TrustDmbPath(EngineVersion version, string fullDmbPath, CancellationToken cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs index f2a60ae5b9f..970e3ab89d5 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs @@ -118,7 +118,7 @@ public EngineManager(IIOManager ioManager, IEngineInstaller engineInstaller, IEv /// public async ValueTask ChangeVersion( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool allowInstallation, @@ -166,8 +166,11 @@ public async ValueTask UseExecutables(EngineVersion? requ "Acquiring lock on BYOND version {version}...", requiredVersion?.ToString() ?? $"{ActiveVersion} (active)"); var versionToUse = requiredVersion ?? ActiveVersion ?? throw new JobException(ErrorCode.EngineNoVersionsInstalled); + + using var progressReporter = new JobProgressReporter(); + var installLock = await AssertAndLockVersion( - null, + progressReporter, versionToUse, null, requiredVersion != null, @@ -388,7 +391,7 @@ await ValueTaskExtensions.WhenAll( /// /// Ensures a BYOND is installed if it isn't already. /// - /// The optional for the operation. + /// The for the operation. /// The to install. /// Optional custom zip file to use. Will cause a number to be added. /// If this BYOND version is required as part of a locking operation. @@ -396,7 +399,7 @@ await ValueTaskExtensions.WhenAll( /// The for the operation. /// A resulting in the . async ValueTask AssertAndLockVersion( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool neededForLock, @@ -443,8 +446,7 @@ async ValueTask AssertAndLockVersion( { if (installedOrInstalling) { - if (progressReporter != null) - progressReporter.StageName = "Waiting for existing installation job..."; + progressReporter.StageName = "Waiting for existing installation job..."; if (neededForLock && !installation.InstallationTask.IsCompleted) logger.LogWarning("The required engine version ({version}) is not readily available! We will have to wait for it to install.", version); @@ -468,8 +470,7 @@ async ValueTask AssertAndLockVersion( else logger.LogInformation("Requested engine version {version} not currently installed. Doing so now...", version); - if (progressReporter != null) - progressReporter.StageName = "Running event"; + progressReporter.StageName = "Running event"; var versionString = version.ToString(); await eventConsumer.HandleEvent(EventType.EngineInstallStart, new List { versionString }, deploymentPipelineProcesses, cancellationToken); @@ -504,14 +505,14 @@ async ValueTask AssertAndLockVersion( /// /// Installs the files for a given BYOND . /// - /// The optional for the operation. + /// The for the operation. /// The being installed with the number set if appropriate. /// Custom zip file to use. Will cause a number to be added. /// If processes should be launched as part of the deployment pipeline. /// The for the operation. /// A representing the running operation. async ValueTask InstallVersionFiles( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool deploymentPipelineProcesses, @@ -528,14 +529,12 @@ async ValueTask DirectoryCleanup() try { IEngineInstallationData engineInstallationData; + var remainingProgress = 1.0; if (customVersionStream == null) { - if (progressReporter != null) - progressReporter.StageName = "Downloading version"; - - engineInstallationData = await engineInstaller.DownloadVersion(version, progressReporter, cancellationToken); - - progressReporter?.ReportProgress(null); + using var subReporter = progressReporter.CreateSection("Downloading Version", 0.5); + remainingProgress -= 0.5; + engineInstallationData = await engineInstaller.DownloadVersion(version, subReporter, cancellationToken); } else #pragma warning disable CA2000 // Dispose objects before losing scope, false positive @@ -544,33 +543,45 @@ async ValueTask DirectoryCleanup() customVersionStream); #pragma warning restore CA2000 // Dispose objects before losing scope - await using (engineInstallationData) + JobProgressReporter remainingReporter; + try + { + remainingReporter = progressReporter.CreateSection(null, remainingProgress); + } + catch { - if (progressReporter != null) - progressReporter.StageName = "Cleaning target directory"; + await engineInstallationData.DisposeAsync(); + throw; + } - await directoryCleanupTask; + using (remainingReporter) + { + await using (engineInstallationData) + { + remainingReporter.StageName = "Cleaning target directory"; - if (progressReporter != null) - progressReporter.StageName = "Extracting data"; + await directoryCleanupTask; + remainingReporter.ReportProgress(0.1); + remainingReporter.StageName = "Extracting data"; - logger.LogTrace("Extracting engine to {extractPath}...", installFullPath); - await engineInstallationData.ExtractToPath(installFullPath, cancellationToken); - } + logger.LogTrace("Extracting engine to {extractPath}...", installFullPath); + await engineInstallationData.ExtractToPath(installFullPath, cancellationToken); + remainingReporter.ReportProgress(0.3); + } - if (progressReporter != null) - progressReporter.StageName = "Running installation actions"; + remainingReporter.StageName = "Running installation actions"; - await engineInstaller.Install(version, installFullPath, deploymentPipelineProcesses, cancellationToken); + await engineInstaller.Install(version, installFullPath, deploymentPipelineProcesses, cancellationToken); - if (progressReporter != null) - progressReporter.StageName = "Writing version file"; + remainingReporter.ReportProgress(0.9); + remainingReporter.StageName = "Writing version file"; - // make sure to do this last because this is what tells us we have a valid version in the future - await ioManager.WriteAllBytes( - ioManager.ConcatPath(installFullPath, VersionFileName), - Encoding.UTF8.GetBytes(version.ToString()), - cancellationToken); + // make sure to do this last because this is what tells us we have a valid version in the future + await ioManager.WriteAllBytes( + ioManager.ConcatPath(installFullPath, VersionFileName), + Encoding.UTF8.GetBytes(version.ToString()), + cancellationToken); + } } catch (HttpRequestException ex) { diff --git a/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs index a4ad57a4c02..0df793e219c 100644 --- a/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs @@ -24,10 +24,10 @@ interface IEngineInstaller /// Download a given engine . /// /// The of the engine to download. - /// The optional for the operation. + /// The for the operation. /// The for the operation. /// A resulting in the for the download. - ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken); + ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken); /// /// Does actions necessary to get an extracted installation working. diff --git a/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs b/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs index 18af1469e6d..7ce4b883db8 100644 --- a/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs +++ b/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs @@ -28,14 +28,14 @@ public interface IEngineManager : IComponentService, IDisposable /// /// Change the active . /// - /// The optional for the operation. + /// The for the operation. /// The new . /// Optional of a custom BYOND version zip file. /// If an installation should be performed if the is not installed. If and an installation is required an will be thrown. /// The for the operation. /// A representing the running operation. ValueTask ChangeVersion( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool allowInstallation, diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index e552d16b406..ac4c0964e92 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -133,23 +133,32 @@ public override IEngineInstallation CreateInstallation(EngineVersion version, st } /// - public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken) + public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken) { CheckVersionValidity(version); + ArgumentNullException.ThrowIfNull(jobProgressReporter); // get a lock on a system wide OD repo Logger.LogTrace("Cloning OD repo..."); - var progressSection1 = jobProgressReporter?.CreateSection("Updating OpenDream git repository", 0.5f); - - var repo = await repositoryManager.CloneRepository( - GeneralConfiguration.OpenDreamGitUrl, - null, - null, - null, - progressSection1, - true, - cancellationToken); + var progressSection1 = jobProgressReporter.CreateSection("Updating OpenDream git repository", 0.5f); + IRepository? repo; + try + { + repo = await repositoryManager.CloneRepository( + GeneralConfiguration.OpenDreamGitUrl, + null, + null, + null, + progressSection1, + true, + cancellationToken); + } + catch + { + progressSection1.Dispose(); + throw; + } try { @@ -157,6 +166,8 @@ public override async ValueTask DownloadVersion(EngineV { Logger.LogTrace("OD repo seems to already exist, attempting load and fetch..."); repo = await repositoryManager.LoadRepository(cancellationToken); + if (repo == null) + throw new JobException("Can't load OpenDream repository! Please delete cache from disk!"); await repo!.FetchOrigin( progressSection1, @@ -166,18 +177,23 @@ public override async ValueTask DownloadVersion(EngineV cancellationToken); } - var progressSection2 = jobProgressReporter?.CreateSection("Checking out OpenDream version", 0.5f); + progressSection1.Dispose(); + progressSection1 = null; - var committish = version.SourceSHA - ?? $"{GeneralConfiguration.OpenDreamGitTagPrefix}{version.Version!.Semver()}"; + using (var progressSection2 = jobProgressReporter.CreateSection("Checking out OpenDream version", 0.5f)) + { + var committish = version.SourceSHA + ?? $"{GeneralConfiguration.OpenDreamGitTagPrefix}{version.Version!.Semver()}"; - await repo.CheckoutObject( - committish, - null, - null, - true, - progressSection2, - cancellationToken); + await repo.CheckoutObject( + committish, + null, + null, + true, + false, + progressSection2, + cancellationToken); + } if (!await repo.CommittishIsParent("tgs-min-compat", cancellationToken)) throw new JobException(ErrorCode.OpenDreamTooOld); @@ -189,6 +205,10 @@ await repo.CheckoutObject( repo?.Dispose(); throw; } + finally + { + progressSection1?.Dispose(); + } } /// diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs index 35bcbf63dc0..ce2c7704e22 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs @@ -224,7 +224,7 @@ public override async ValueTask TrustDmbPath(EngineVersion version, string fullD /// protected override string GetDreamDaemonName(Version byondVersion, out bool supportsCli) { - supportsCli = byondVersion >= DDExeVersion; + supportsCli = byondVersion >= DDExeVersion && !sessionConfiguration.ForceUseDreamDaemonExe; return supportsCli ? "dd.exe" : "dreamdaemon.exe"; } diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index 5d8c0b9054d..711d6ca246d 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -12,7 +12,7 @@ public enum EventType RepoResetOrigin, /// - /// Parameters: Checkout target. + /// Parameters: Checkout target, hard reset flag (If "True", this is actually a hard reset, not a checkout). /// [EventScript("RepoCheckout")] RepoCheckout, diff --git a/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs b/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs index 6109f490bb7..69289bb5d40 100644 --- a/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs +++ b/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs @@ -49,8 +49,8 @@ public GitHubRemoteFeatures(IGitHubServiceFactory gitHubServiceFactory, ILogger< CancellationToken cancellationToken) { var gitHubService = repositorySettings.AccessToken != null - ? gitHubServiceFactory.CreateService(repositorySettings.AccessToken) - : gitHubServiceFactory.CreateService(); + ? await gitHubServiceFactory.CreateService(repositorySettings.AccessToken, cancellationToken) + : await gitHubServiceFactory.CreateService(cancellationToken); PullRequest? pr = null; ApiException? exception = null; diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs index 65bcd1d61ed..74417818f19 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs @@ -47,7 +47,8 @@ public interface IRepository : IGitRemoteAdditionalInformation, IDisposable /// The optional username used for fetching from submodule repositories. /// The optional password used for fetching from submodule repositories. /// If a submodule update should be attempted after the merge. - /// The optional to report progress of the operation. + /// If a hard reset to the target committish should be performed instead of a checkout. + /// The to report progress of the operation. /// The for the operation. /// A representing the running operation. ValueTask CheckoutObject( @@ -55,7 +56,8 @@ ValueTask CheckoutObject( string? username, string? password, bool updateSubmodules, - JobProgressReporter? progressReporter, + bool moveCurrentReference, + JobProgressReporter progressReporter, CancellationToken cancellationToken); /// @@ -83,14 +85,14 @@ ValueTask AddTestMerge( /// /// Fetch commits from the origin repository. /// - /// The optional to report progress of the operation. + /// The to report progress of the operation. /// The optional username to fetch from the origin repository. /// The optional password to fetch from the origin repository. /// If any events created should be marked as part of the deployment pipeline. /// The for the operation. /// A representing the running operation. ValueTask FetchOrigin( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, string? username, string? password, bool deploymentPipeline, diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs index 596301611ac..87cbe63d95a 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs @@ -35,7 +35,7 @@ public interface IRepositoryManager : IDisposable /// The optional branch to clone. /// The optional username to clone from . /// The optional password to clone from . - /// The optional for progress of the clone. + /// The for progress of the clone. /// If submodules should be recusively cloned and initialized. /// The for the operation. /// A resulting i the newly cloned , if one already exists. @@ -44,7 +44,7 @@ public interface IRepositoryManager : IDisposable string? initialBranch, string? username, string? password, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, bool recurseSubmodules, CancellationToken cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 972fae31cca..9a71bcdcebf 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -232,13 +232,14 @@ await Task.Factory.StartNew( logger.LogTrace("Fetching refspec {refSpec}...", refSpec); var remote = libGitRepo.Network.Remotes.First(); + using var fetchReporter = progressReporter.CreateSection($"Fetch {refSpec}", progressFactor); commands.Fetch( libGitRepo, refSpecList, remote, new FetchOptions().Hydrate( logger, - progressReporter.CreateSection($"Fetch {refSpec}", progressFactor), + fetchReporter, credentialsProvider.GenerateCredentialsHandler(username, password), cancellationToken), logMessage); @@ -267,14 +268,14 @@ await Task.Factory.StartNew( logger.LogTrace("Merging {targetCommitSha} into {currentReference}...", testMergeParameters.TargetCommitSha[..7], Reference); + using var mergeReporter = progressReporter.CreateSection($"Merge {testMergeParameters.TargetCommitSha[..7]}", progressFactor); result = libGitRepo.Merge(testMergeParameters.TargetCommitSha, sig, new MergeOptions { CommitOnSuccess = commitMessage == null, FailOnConflict = false, // Needed to get conflicting files FastForwardStrategy = FastForwardStrategy.NoFastForward, SkipReuc = true, - OnCheckoutProgress = CheckoutProgressHandler( - progressReporter.CreateSection($"Merge {testMergeParameters.TargetCommitSha[..7]}", progressFactor)), + OnCheckoutProgress = CheckoutProgressHandler(mergeReporter), }); } finally @@ -295,7 +296,8 @@ await Task.Factory.StartNew( var revertTo = originalCommit.CanonicalName ?? originalCommit.Tip.Sha; logger.LogDebug("Merge conflict, aborting and reverting to {revertTarget}", revertTo); progressReporter.ReportProgress(0); - RawCheckout(revertTo, progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0), cancellationToken); + using var revertReporter = progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0); + RawCheckout(revertTo, false, revertReporter, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); } @@ -343,8 +345,9 @@ await Task.Factory.StartNew( if (updateSubmodules) { + using var progressReporter2 = progressReporter.CreateSection("Update Submodules", progressFactor); await UpdateSubmodules( - progressReporter.CreateSection("Update Submodules", progressFactor), + progressReporter2, username, password, false, @@ -376,20 +379,23 @@ public async ValueTask CheckoutObject( string? username, string? password, bool updateSubmodules, - JobProgressReporter? progressReporter, + bool moveCurrentReference, + JobProgressReporter progressReporter, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(committish); logger.LogDebug("Checkout object: {committish}...", committish); - await eventConsumer.HandleEvent(EventType.RepoCheckout, new List { committish }, false, cancellationToken); + await eventConsumer.HandleEvent(EventType.RepoCheckout, new List { committish, moveCurrentReference.ToString() }, false, cancellationToken); await Task.Factory.StartNew( () => { libGitRepo.RemoveUntrackedFiles(); + using var progressReporter3 = progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0); RawCheckout( committish, - progressReporter?.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0), + moveCurrentReference, + progressReporter3, cancellationToken); }, cancellationToken, @@ -397,17 +403,20 @@ await Task.Factory.StartNew( TaskScheduler.Current); if (updateSubmodules) + { + using var progressReporter2 = progressReporter.CreateSection(null, 1.0 / 3); await UpdateSubmodules( - progressReporter?.CreateSection(null, 1.0 / 3), + progressReporter2, username, password, false, cancellationToken); + } } /// public async ValueTask FetchOrigin( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, string? username, string? password, bool deploymentPipeline, @@ -421,13 +430,14 @@ await Task.Factory.StartNew( var remote = libGitRepo.Network.Remotes.First(); try { + using var subReporter = progressReporter.CreateSection("Fetch Origin", 1.0); var fetchOptions = new FetchOptions { Prune = true, TagFetchMode = TagFetchMode.All, }.Hydrate( logger, - progressReporter?.CreateSection("Fetch Origin", 1.0), + subReporter, credentialsProvider.GenerateCredentialsHandler(username, password), cancellationToken); @@ -469,18 +479,23 @@ public async ValueTask ResetToOrigin( logger.LogTrace("Reset to origin..."); var trackedBranch = libGitRepo.Head.TrackedBranch; await eventConsumer.HandleEvent(EventType.RepoResetOrigin, new List { trackedBranch.FriendlyName, trackedBranch.Tip.Sha }, deploymentPipeline, cancellationToken); - await ResetToSha( - trackedBranch.Tip.Sha, - progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0), - cancellationToken); + + using (var progressReporter2 = progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0)) + await ResetToSha( + trackedBranch.Tip.Sha, + progressReporter2, + cancellationToken); if (updateSubmodules) + { + using var progressReporter3 = progressReporter.CreateSection(null, 1.0 / 3); await UpdateSubmodules( - progressReporter.CreateSection(null, 1.0 / 3), + progressReporter3, username, password, deploymentPipeline, cancellationToken); + } } /// @@ -682,9 +697,10 @@ await eventConsumer.HandleEvent( await Task.Factory.StartNew( () => { + using var resetProgress = progressReporter.CreateSection("Hard reset and remove untracked files", 0.1); libGitRepo.Reset(ResetMode.Hard, libGitRepo.Head.Tip, new CheckoutOptions { - OnCheckoutProgress = CheckoutProgressHandler(progressReporter.CreateSection("Hard reset and remove untracked files", 0.1)), + OnCheckoutProgress = CheckoutProgressHandler(resetProgress), }); cancellationToken.ThrowIfCancellationRequested(); libGitRepo.RemoveUntrackedFiles(); @@ -697,10 +713,11 @@ await Task.Factory.StartNew( var remainingProgressFactor = 0.9; if (!synchronizeTrackedBranch) { + using var progressReporter2 = progressReporter.CreateSection("Push to temporary branch", remainingProgressFactor); await PushHeadToTemporaryBranch( username, password, - progressReporter.CreateSection("Push to temporary branch", remainingProgressFactor), + progressReporter2, cancellationToken); return false; } @@ -720,13 +737,24 @@ await PushHeadToTemporaryBranch( var remote = libGitRepo.Network.Remotes.First(); try { - libGitRepo.Network.Push( - libGitRepo.Head, - GeneratePushOptions( - progressReporter.CreateSection("Push to origin", remainingProgressFactor), - username, - password, - cancellationToken)); + using var pushReporter = progressReporter.CreateSection("Push to origin", remainingProgressFactor); + var (pushOptions, progressReporters) = GeneratePushOptions( + pushReporter, + username, + password, + cancellationToken); + try + { + libGitRepo.Network.Push( + libGitRepo.Head, + pushOptions); + } + finally + { + foreach (var progressReporter in progressReporters) + progressReporter.Dispose(); + } + return true; } catch (NonFastForwardException) @@ -879,9 +907,10 @@ protected override void DisposeImpl() /// Runs a blocking force checkout to . /// /// The committish to checkout. - /// The optional for the operation. + /// If a hard reset should actually be performed. + /// The for the operation. /// The for the operation. - void RawCheckout(string committish, JobProgressReporter? progressReporter, CancellationToken cancellationToken) + void RawCheckout(string committish, bool moveCurrentReference, JobProgressReporter progressReporter, CancellationToken cancellationToken) { logger.LogTrace("Checkout: {committish}", committish); @@ -890,44 +919,59 @@ void RawCheckout(string committish, JobProgressReporter? progressReporter, Cance CheckoutModifiers = CheckoutModifiers.Force, }; - if (progressReporter != null) - { - var stage = $"Checkout {committish}"; - progressReporter = progressReporter.CreateSection(stage, 1.0); - progressReporter.ReportProgress(0); - checkoutOptions.OnCheckoutProgress = CheckoutProgressHandler(progressReporter); - } + var stage = $"Checkout {committish}"; + using var newProgressReporter = progressReporter.CreateSection(stage, 1.0); + newProgressReporter.ReportProgress(0); + checkoutOptions.OnCheckoutProgress = CheckoutProgressHandler(newProgressReporter); cancellationToken.ThrowIfCancellationRequested(); - void RunCheckout() => commands.Checkout( - libGitRepo, - checkoutOptions, - committish); - - try + if (moveCurrentReference) { - RunCheckout(); + if (Reference == NoReference) + throw new InvalidOperationException("Cannot move current reference when not on reference!"); + + var gitObject = libGitRepo.Lookup(committish); + if (gitObject == null) + throw new JobException($"Could not find committish: {committish}"); + + var commit = gitObject.Peel(); + + cancellationToken.ThrowIfCancellationRequested(); + + libGitRepo.Reset(ResetMode.Hard, commit, checkoutOptions); } - catch (NotFoundException) + else { - // Maybe (likely) a remote? - var remoteName = $"origin/{committish}"; - var remoteBranch = libGitRepo.Branches.FirstOrDefault( - branch => branch.FriendlyName.Equals(remoteName, StringComparison.Ordinal)); - cancellationToken.ThrowIfCancellationRequested(); + void RunCheckout() => commands.Checkout( + libGitRepo, + checkoutOptions, + committish); - if (remoteBranch == default) - throw; + try + { + RunCheckout(); + } + catch (NotFoundException) + { + // Maybe (likely) a remote? + var remoteName = $"origin/{committish}"; + var remoteBranch = libGitRepo.Branches.FirstOrDefault( + branch => branch.FriendlyName.Equals(remoteName, StringComparison.Ordinal)); + cancellationToken.ThrowIfCancellationRequested(); - logger.LogDebug("Creating local branch for {remoteBranchFriendlyName}...", remoteBranch.FriendlyName); - var branch = libGitRepo.CreateBranch(committish, remoteBranch.Tip); + if (remoteBranch == default) + throw; - libGitRepo.Branches.Update(branch, branchUpdate => branchUpdate.TrackedBranch = remoteBranch.CanonicalName); + logger.LogDebug("Creating local branch for {remoteBranchFriendlyName}...", remoteBranch.FriendlyName); + var branch = libGitRepo.CreateBranch(committish, remoteBranch.Tip); - cancellationToken.ThrowIfCancellationRequested(); + libGitRepo.Branches.Update(branch, branchUpdate => branchUpdate.TrackedBranch = remoteBranch.CanonicalName); - RunCheckout(); + cancellationToken.ThrowIfCancellationRequested(); + + RunCheckout(); + } } cancellationToken.ThrowIfCancellationRequested(); @@ -955,9 +999,38 @@ Task PushHeadToTemporaryBranch(string username, string password, JobProgressRepo try { var forcePushString = String.Format(CultureInfo.InvariantCulture, "+{0}:{0}", branch.CanonicalName); - libGitRepo.Network.Push(remote, forcePushString, GeneratePushOptions(progressReporter.CreateSection(null, 0.9), username, password, cancellationToken)); + + using (var mainPushReporter = progressReporter.CreateSection(null, 0.9)) + { + var (pushOptions, progressReporters) = GeneratePushOptions( + mainPushReporter, + username, + password, + cancellationToken); + + try + { + libGitRepo.Network.Push(remote, forcePushString, pushOptions); + } + finally + { + foreach (var progressReporter in progressReporters) + progressReporter.Dispose(); + } + } + var removalString = String.Format(CultureInfo.InvariantCulture, ":{0}", branch.CanonicalName); - libGitRepo.Network.Push(remote, removalString, GeneratePushOptions(progressReporter.CreateSection(null, 0.1), username, password, cancellationToken)); + using var forcePushReporter = progressReporter.CreateSection(null, 0.1); + var (forcePushOptions, forcePushReporters) = GeneratePushOptions(forcePushReporter, username, password, cancellationToken); + try + { + libGitRepo.Network.Push(remote, removalString, forcePushOptions); + } + finally + { + foreach (var subForcePushReporter in forcePushReporters) + forcePushReporter.Dispose(); + } } catch (UserCancelledException) { @@ -984,32 +1057,39 @@ Task PushHeadToTemporaryBranch(string username, string password, JobProgressRepo /// The username for the . /// The password for the . /// The for the operation. - /// A new set of . - PushOptions GeneratePushOptions(JobProgressReporter progressReporter, string username, string password, CancellationToken cancellationToken) + /// A new set of and the associated s based off . + (PushOptions PushOptions, IEnumerable SubProgressReporters) GeneratePushOptions(JobProgressReporter progressReporter, string username, string password, CancellationToken cancellationToken) { - var subProgressReporter = progressReporter.CreateSection(null, 0.5); + var packFileCountingReporter = progressReporter.CreateSection(null, 0.25); + var packFileDeltafyingReporter = progressReporter.CreateSection(null, 0.25); + var transferProgressReporter = progressReporter.CreateSection(null, 0.5); - return new PushOptions - { - OnPackBuilderProgress = (stage, current, total) => - { - var baseProgress = stage == PackBuilderStage.Counting ? 0 : 0.5; - var addon = total > 0 && current <= total ? (0.5 * ((double)current / total)) : 0; - progressReporter.ReportProgress(baseProgress + addon); - return !cancellationToken.IsCancellationRequested; - }, - OnNegotiationCompletedBeforePush = (a) => + return ( + PushOptions: new PushOptions { - subProgressReporter = progressReporter.CreateSection(null, 0.5); - return !cancellationToken.IsCancellationRequested; + OnPackBuilderProgress = (stage, current, total) => + { + if (total < current) + total = current; + + var percentage = ((double)current) / total; + (stage == PackBuilderStage.Counting ? packFileCountingReporter : packFileDeltafyingReporter).ReportProgress(percentage); + return !cancellationToken.IsCancellationRequested; + }, + OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, + OnPushTransferProgress = (a, sentBytes, totalBytes) => + { + packFileCountingReporter.ReportProgress((double)sentBytes / totalBytes); + return !cancellationToken.IsCancellationRequested; + }, + CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), }, - OnPushTransferProgress = (a, sentBytes, totalBytes) => + SubProgressReporters: new List { - progressReporter.ReportProgress((double)sentBytes / totalBytes); - return !cancellationToken.IsCancellationRequested; - }, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), - }; + packFileCountingReporter, + packFileDeltafyingReporter, + transferProgressReporter, + }); } /// @@ -1033,7 +1113,7 @@ string GetRepositoryPath() /// The for the operation. /// A representing the running operation. ValueTask UpdateSubmodules( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, string? username, string? password, bool deploymentPipeline, @@ -1041,7 +1121,7 @@ ValueTask UpdateSubmodules( { logger.LogTrace("Updating submodules {withOrWithout} credentials...", username == null ? "without" : "with"); - async ValueTask RecursiveUpdateSubmodules(LibGit2Sharp.IRepository parentRepository, JobProgressReporter? currentProgressReporter, string parentGitDirectory) + async ValueTask RecursiveUpdateSubmodules(LibGit2Sharp.IRepository parentRepository, JobProgressReporter currentProgressReporter, string parentGitDirectory) { var submoduleCount = libGitRepo.Submodules.Count(); if (submoduleCount == 0) @@ -1060,15 +1140,16 @@ async ValueTask RecursiveUpdateSubmodules(LibGit2Sharp.IRepository parentReposit OnCheckoutNotify = (_, _) => !cancellationToken.IsCancellationRequested, }; + using var fetchReporter = currentProgressReporter.CreateSection($"Fetch submodule {submodule.Name}", factor); + submoduleUpdateOptions.FetchOptions.Hydrate( logger, - currentProgressReporter?.CreateSection($"Fetch submodule {submodule.Name}", factor), + fetchReporter, credentialsProvider.GenerateCredentialsHandler(username, password), cancellationToken); - if (currentProgressReporter != null) - submoduleUpdateOptions.OnCheckoutProgress = CheckoutProgressHandler( - currentProgressReporter.CreateSection($"Checkout submodule {submodule.Name}", factor)); + using var checkoutReporter = currentProgressReporter.CreateSection($"Checkout submodule {submodule.Name}", factor); + submoduleUpdateOptions.OnCheckoutProgress = CheckoutProgressHandler(checkoutReporter); logger.LogDebug("Updating submodule {submoduleName}...", submodule.Name); Task RawSubModuleUpdate() => Task.Factory.StartNew( @@ -1085,7 +1166,7 @@ Task RawSubModuleUpdate() => Task.Factory.StartNew( { // workaround for https://github.com/libgit2/libgit2/issues/3820 // kill off the modules/ folder in .git and try again - currentProgressReporter?.ReportProgress(null); + currentProgressReporter.ReportProgress(0); credentialsProvider.CheckBadCredentialsException(ex); logger.LogWarning(ex, "Initial update of submodule {submoduleName} failed. Deleting submodule directories and re-attempting...", submodule.Name); @@ -1124,9 +1205,11 @@ await eventConsumer.HandleEvent( using var submoduleRepo = await submoduleFactory.CreateFromPath( submodulePath, cancellationToken); + + using var submoduleReporter = currentProgressReporter.CreateSection($"Entering submodule \"{submodule.Name}\"...", factor); await RecursiveUpdateSubmodules( submoduleRepo, - currentProgressReporter?.CreateSection($"Entering submodule \"{submodule.Name}\"...", factor), + submoduleReporter, submodulePath); } } diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 94ae1769510..74a6bc67514 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -123,7 +123,7 @@ public void Dispose() string? initialBranch, string? username, string? password, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, bool recurseSubmodules, CancellationToken cancellationToken) { @@ -146,8 +146,8 @@ public void Dispose() if (!await ioManager.DirectoryExists(repositoryPath, cancellationToken)) try { - var cloneProgressReporter = progressReporter?.CreateSection(null, 0.75f); - var checkoutProgressReporter = progressReporter?.CreateSection(null, 0.25f); + using var cloneProgressReporter = progressReporter.CreateSection(null, 0.75f); + using var checkoutProgressReporter = progressReporter.CreateSection(null, 0.25f); var cloneOptions = new CloneOptions { RecurseSubmodules = recurseSubmodules, diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs index a7aca78c4b8..17e820433a7 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs @@ -22,11 +22,6 @@ namespace Tgstation.Server.Host.Components.Repository /// sealed class RepositoryUpdateService { - /// - /// The for the . - /// - readonly RepositoryUpdateRequest model; - /// /// The current for the . /// @@ -50,19 +45,16 @@ sealed class RepositoryUpdateService /// /// Initializes a new instance of the class. /// - /// The value of . /// The value of . /// The value of . /// The value of . /// The value of . public RepositoryUpdateService( - RepositoryUpdateRequest model, RepositorySettings currentModel, User initiatingUser, ILogger logger, long instanceId) { - this.model = model ?? throw new ArgumentNullException(nameof(model)); this.currentModel = currentModel ?? throw new ArgumentNullException(nameof(currentModel)); this.initiatingUser = initiatingUser ?? throw new ArgumentNullException(nameof(initiatingUser)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -139,24 +131,25 @@ public static async ValueTask LoadRevisionInformation( /// /// The job entrypoint used by to update the repository's current HEAD. /// - /// The the job is running on. only when performing an instance move operation. + /// The . + /// The the job is running on. /// The for the operation. - /// The running , ignored. /// The for the job. /// The for the operation. /// A representing the running operation. #pragma warning disable CA1502, CA1506 // TODO: Decomplexify public async ValueTask RepositoryUpdateJob( + RepositoryUpdateRequest model, IInstanceCore? instance, IDatabaseContextFactory databaseContextFactory, - Job job, JobProgressReporter progressReporter, CancellationToken cancellationToken) #pragma warning restore CA1502, CA1506 { + ArgumentNullException.ThrowIfNull(model); ArgumentNullException.ThrowIfNull(instance); - - _ = job; // shuts up an IDE warning + ArgumentNullException.ThrowIfNull(databaseContextFactory); + ArgumentNullException.ThrowIfNull(progressReporter); var repoManager = instance.RepositoryManager; using var repo = await repoManager.LoadRepository(cancellationToken) ?? throw new JobException(ErrorCode.RepoMissing); @@ -180,10 +173,7 @@ public async ValueTask RepositoryUpdateJob( var numSteps = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0) + (!modelHasShaOrReference ? 2 : (hardResettingToOriginReference ? 3 : 1)); var progressFactor = 1.0 / numSteps; - JobProgressReporter NextProgressReporter(string? stage) - { - return progressReporter.CreateSection(stage, progressFactor); - } + JobProgressReporter NextProgressReporter(string? stage) => progressReporter.CreateSection(stage, progressFactor); progressReporter.ReportProgress(0); @@ -253,29 +243,35 @@ ValueTask CallLoadRevInfo(Models.TestMerge? testMergeToAdd = null, string? lastO { if (!repo.Tracking) throw new JobException(ErrorCode.RepoReferenceRequired); - await repo.FetchOrigin( - NextProgressReporter("Fetch Origin"), - currentModel.AccessUser, - currentModel.AccessToken, - false, - cancellationToken); + using (var fetchReporter = NextProgressReporter("Fetch Origin")) + await repo.FetchOrigin( + fetchReporter, + currentModel.AccessUser, + currentModel.AccessToken, + false, + cancellationToken); if (!modelHasShaOrReference) { - var fastForward = await repo.MergeOrigin( - NextProgressReporter("Merge Origin"), - committerName, - currentModel.CommitterEmail!, - false, - cancellationToken); + bool? fastForward; + using (var mergeReporter = NextProgressReporter("Merge Origin")) + fastForward = await repo.MergeOrigin( + mergeReporter, + committerName, + currentModel.CommitterEmail!, + false, + cancellationToken); + if (!fastForward.HasValue) throw new JobException(ErrorCode.RepoMergeConflict); + lastRevisionInfo!.OriginCommitSha = await repo.GetOriginSha(cancellationToken); await UpdateRevInfo(); if (fastForward.Value) { + using var syncReporter = NextProgressReporter("Sychronize"); await repo.Synchronize( - NextProgressReporter("Sychronize"), + syncReporter, currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName!, @@ -286,7 +282,7 @@ await repo.Synchronize( postUpdateSha = repo.Head; } else - NextProgressReporter(null).ReportProgress(1.0); + NextProgressReporter(null).Dispose(); } } @@ -310,38 +306,44 @@ await repo.Synchronize( if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null)) throw new JobException(ErrorCode.RepoSwappedShaOrReference); - await repo.CheckoutObject( - committish, - currentModel.AccessUser, - currentModel.AccessToken, - updateSubmodules, - NextProgressReporter("Checkout"), - cancellationToken); + using (var checkoutReporter = NextProgressReporter("Checkout")) + await repo.CheckoutObject( + committish, + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + false, + checkoutReporter, + cancellationToken); await CallLoadRevInfo(); // we've either seen origin before or what we're checking out is on origin } else - NextProgressReporter(null).ReportProgress(1.0); + NextProgressReporter(null).Dispose(); if (hardResettingToOriginReference) { if (!repo.Tracking) throw new JobException(ErrorCode.RepoReferenceNotTracking); - await repo.ResetToOrigin( - NextProgressReporter("Reset to Origin"), - currentModel.AccessUser, - currentModel.AccessToken, - updateSubmodules, - false, - cancellationToken); - await repo.Synchronize( - NextProgressReporter("Synchronize"), - currentModel.AccessUser, - currentModel.AccessToken, - currentModel.CommitterName!, - currentModel.CommitterEmail!, - true, - false, - cancellationToken); + using (var resetReporter = NextProgressReporter("Reset to Origin")) + await repo.ResetToOrigin( + resetReporter, + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + false, + cancellationToken); + + using (var syncReporter = NextProgressReporter("Synchronize")) + await repo.Synchronize( + syncReporter, + currentModel.AccessUser, + currentModel.AccessToken, + currentModel.CommitterName!, + currentModel.CommitterEmail!, + true, + false, + cancellationToken); + await CallLoadRevInfo(); // repo head is on origin so force this @@ -492,7 +494,8 @@ await databaseContextFactory.UseContext( // goteem var commitSha = revInfoWereLookingFor.CommitSha!; logger.LogDebug("Reusing existing SHA {sha}...", commitSha); - await repo.ResetToSha(commitSha, NextProgressReporter($"Reset to {commitSha[..7]}"), cancellationToken); + using var resetReporter = NextProgressReporter($"Reset to {commitSha[..7]}"); + await repo.ResetToSha(commitSha, resetReporter, cancellationToken); lastRevisionInfo = revInfoWereLookingFor; } @@ -505,15 +508,17 @@ await databaseContextFactory.UseContext( var fullTestMergeTask = repo.GetTestMerge(newTestMerge, currentModel, cancellationToken); - var mergeResult = await repo.AddTestMerge( - newTestMerge, - committerName, - currentModel.CommitterEmail!, - currentModel.AccessUser, - currentModel.AccessToken, - updateSubmodules, - NextProgressReporter($"Test merge #{newTestMerge.Number}"), - cancellationToken); + TestMergeResult mergeResult; + using (var testMergeReporter = NextProgressReporter($"Test merge #{newTestMerge.Number}")) + mergeResult = await repo.AddTestMerge( + newTestMerge, + committerName, + currentModel.CommitterEmail!, + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + testMergeReporter, + cancellationToken); if (mergeResult.Status == MergeStatus.Conflicts) throw new JobException( @@ -552,15 +557,17 @@ await databaseContextFactory.UseContext( var currentHead = repo.Head; if (currentModel.PushTestMergeCommits!.Value && (startSha != currentHead || (postUpdateSha != null && postUpdateSha != currentHead))) { - await repo.Synchronize( - NextProgressReporter("Synchronize"), - currentModel.AccessUser, - currentModel.AccessToken, - currentModel.CommitterName!, - currentModel.CommitterEmail!, - false, - false, - cancellationToken); + using (var syncReporter = NextProgressReporter("Synchronize")) + await repo.Synchronize( + syncReporter, + currentModel.AccessUser, + currentModel.AccessToken, + currentModel.CommitterName!, + currentModel.CommitterEmail!, + false, + false, + cancellationToken); + await UpdateRevInfo(); } } @@ -574,19 +581,109 @@ await repo.Synchronize( var secondStep = startReference != null && repo.Head != startSha; // DCTx2: Cancellation token is for job, operations should always run - await repo.CheckoutObject( - startReference ?? startSha, + using (var checkoutReporter = progressReporter.CreateSection($"Checkout {startReference ?? startSha[..7]}", secondStep ? 0.5 : 1.0)) + await repo.CheckoutObject( + startReference ?? startSha, + currentModel.AccessUser, + currentModel.AccessToken, + true, + false, + checkoutReporter, + default); + + if (secondStep) + using (var resetReporter = progressReporter.CreateSection($"Hard reset to SHA {startSha[..7]}", 0.5)) + await repo.ResetToSha(startSha, resetReporter, default); + + throw; + } + } + + /// + /// The job entrypoint used by to reclone a repository. + /// + /// The the job is running on. + /// The for the operation. + /// The for the job. + /// The for the operation. + /// A representing the running operation. + public async ValueTask RepositoryRecloneJob( + IInstanceCore? instance, + IDatabaseContextFactory databaseContextFactory, + JobProgressReporter progressReporter, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(instance); + ArgumentNullException.ThrowIfNull(databaseContextFactory); + ArgumentNullException.ThrowIfNull(progressReporter); + + progressReporter.StageName = "Loading Old Repository"; + + Uri origin; + string? oldReference; + string oldSha; + ValueTask deleteTask; + using (var deleteReporter = progressReporter.CreateSection("Deleting Old Repository", 0.1)) + { + using (var oldRepo = await instance.RepositoryManager.LoadRepository(cancellationToken)) + { + if (oldRepo == null) + throw new JobException(ErrorCode.RepoMissing); + + origin = oldRepo.Origin; + oldSha = oldRepo.Head; + oldReference = oldRepo.Reference; + if (oldReference == Repository.NoReference) + oldReference = null; + + deleteTask = instance.RepositoryManager.DeleteRepository(cancellationToken); + } + + await deleteTask; + } + + IRepository newRepo; + try + { + using var cloneReporter = progressReporter.CreateSection("Cloning New Repository", 0.8); + newRepo = await instance.RepositoryManager.CloneRepository( + origin, + oldReference, currentModel.AccessUser, currentModel.AccessToken, - true, - progressReporter.CreateSection($"Checkout {startReference ?? startSha[..7]}", secondStep ? 0.5 : 1.0), - default); + cloneReporter, + true, // TODO: Make configurable maybe... + cancellationToken) + ?? throw new JobException("A race condition occurred while recloning the repository. Somehow, it was fully cloned instantly after being deleted!"); // I'll take lines of code that should never be hit for $10k + } + catch (Exception ex) when (ex is not JobException) + { + logger.LogWarning("Reclone failed, clearing credentials!"); - if (secondStep) - await repo.ResetToSha(startSha, progressReporter.CreateSection($"Hard reset to SHA {startSha[..7]}", 0.5), default); + // need to clear credentials here + await databaseContextFactory.UseContextTaskReturn(context => + { + context.RepositorySettings.Attach(currentModel); + currentModel.AccessUser = null; + currentModel.AccessToken = null; + return context.Save(CancellationToken.None); // DCT: Must always run + }); throw; } + + using (newRepo) + using (var checkoutReporter = progressReporter.CreateSection("Checking out previous Detached Commit", 0.1)) + { + await newRepo.CheckoutObject( + oldSha, + currentModel.AccessUser, + currentModel.AccessToken, + false, + oldReference != null, + checkoutReporter, + cancellationToken); + } } } } diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 10556ca9fc2..754b672b7d6 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -121,6 +121,9 @@ async Task Wrap() /// public FifoSemaphore TopicSendSemaphore { get; } + /// + public long? MemoryUsage => process.MemoryUsage; + /// /// The for the . /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs index 96c6c9ccf9a..2220c569327 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs @@ -24,6 +24,11 @@ public interface IWatchdog : IComponentService, IAsyncDisposable, IEventConsumer /// WatchdogStatus Status { get; } + /// + /// Gets the memory usage of the game server in bytes. + /// + long? MemoryUsage { get; } + /// /// If the alpha server is the active server. /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 28156db6e5e..5d9326ce7dd 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -50,6 +50,9 @@ protected set } } + /// + public long? MemoryUsage => GetActiveController()?.MemoryUsage; + /// public abstract bool AlphaIsActive { get; } diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index bc9c11e2f36..1790316e370 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -89,7 +89,7 @@ public sealed class GeneralConfiguration : ServerInformationBase public ushort ApiPort { get; set; } /// - /// A GitHub personal access token to use for bypassing rate limits on requests. Requires no scopes. + /// A classic GitHub personal access token to use for bypassing rate limits on requests. Requires no scopes. /// public string? GitHubAccessToken { get; set; } diff --git a/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs b/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs index c543698b905..d7ee3e1c9d2 100644 --- a/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs @@ -29,5 +29,10 @@ sealed class SessionConfiguration /// If , deployments that fail will not be immediately cleaned up. They will be cleaned up the next time the instance is onlined. /// public bool DelayCleaningFailedDeployments { get; set; } + + /// + /// If set dd.exe will not be used on Windows systems in versions where it is present. Instead dreamdaemon.exe will always be used. + /// + public bool ForceUseDreamDaemonExe { get; set; } } } diff --git a/src/Tgstation.Server.Host/Configuration/TelemetryConfiguration.cs b/src/Tgstation.Server.Host/Configuration/TelemetryConfiguration.cs new file mode 100644 index 00000000000..7ec0361759d --- /dev/null +++ b/src/Tgstation.Server.Host/Configuration/TelemetryConfiguration.cs @@ -0,0 +1,33 @@ +namespace Tgstation.Server.Host.Configuration +{ + /// + /// Configuration options for telemetry. + /// + public sealed class TelemetryConfiguration + { + /// + /// The key for the the resides in. + /// + public const string Section = "Telemetry"; + + /// + /// The default value of . + /// + private const long DefaultVersionReportingRepositoryId = 841149827; // https://github.com/tgstation/tgstation-server-deployments + + /// + /// If version reporting telemetry is disabled. + /// + public bool DisableVersionReporting { get; set; } + + /// + /// The friendly name used on GitHub deployments for version reporting. If only the server will be shown. + /// + public string? ServerFriendlyName { get; set; } + + /// + /// The GitHub repository ID used for version reporting. + /// + public long? VersionReportingRepositoryId { get; set; } = DefaultVersionReportingRepositoryId; + } +} diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index 2ccc4f4c8bc..c41113069d2 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -7,6 +7,7 @@ using System.Web; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -42,6 +43,11 @@ public sealed class AdministrationController : ApiController /// const string OctokitException = "Bad GitHub API response, check configuration!"; + /// + /// The key for . + /// + static readonly object ReadCacheKey = new(); + /// /// The for the . /// @@ -77,6 +83,11 @@ public sealed class AdministrationController : ApiController /// readonly IFileTransferTicketProvider fileTransferService; + /// + /// The for the . + /// + readonly IMemoryCache cacheService; + /// /// The for the . /// @@ -94,6 +105,7 @@ public sealed class AdministrationController : ApiController /// The value of . /// The value of . /// The value of . + /// The value of . /// The for the . /// The containing value of . /// The for the . @@ -107,6 +119,7 @@ public AdministrationController( IIOManager ioManager, IPlatformIdentifier platformIdentifier, IFileTransferTicketProvider fileTransferService, + IMemoryCache cacheService, ILogger logger, IOptions fileLoggingConfigurationOptions, IApiHeadersProvider apiHeadersProvider) @@ -124,12 +137,14 @@ public AdministrationController( this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); + this.cacheService = cacheService ?? throw new ArgumentNullException(nameof(cacheService)); fileLoggingConfiguration = fileLoggingConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(fileLoggingConfigurationOptions)); } /// /// Get server information. /// + /// If , the cache should be bypassed. /// The for the operation. /// A resulting in the for the operation. /// Retrieved data successfully. @@ -140,39 +155,56 @@ public AdministrationController( [ProducesResponseType(typeof(AdministrationResponse), 200)] [ProducesResponseType(typeof(ErrorMessageResponse), 424)] [ProducesResponseType(typeof(ErrorMessageResponse), 429)] - public async ValueTask Read(CancellationToken cancellationToken) + public async ValueTask Read([FromQuery] bool? fresh, CancellationToken cancellationToken) { try { - Version? greatestVersion = null; - Uri? repoUrl = null; - try + async Task CacheFactory() { - var gitHubService = gitHubServiceFactory.CreateService(); - var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken); - var releases = await gitHubService.GetTgsReleases(cancellationToken); - - foreach (var kvp in releases) + Version? greatestVersion = null; + Uri? repoUrl = null; + try + { + var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); + var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken); + var releases = await gitHubService.GetTgsReleases(cancellationToken); + + foreach (var kvp in releases) + { + var version = kvp.Key; + var release = kvp.Value; + if (version.Major > 3 // Forward/backward compatible but not before TGS4 + && (greatestVersion == null || version > greatestVersion)) + greatestVersion = version; + } + + repoUrl = await repositoryUrlTask; + } + catch (NotFoundException e) { - var version = kvp.Key; - var release = kvp.Value; - if (version.Major > 3 // Forward/backward compatible but not before TGS4 - && (greatestVersion == null || version > greatestVersion)) - greatestVersion = version; + Logger.LogWarning(e, "Not found exception while retrieving upstream repository info!"); } - repoUrl = await repositoryUrlTask; + return Json(new AdministrationResponse + { + LatestVersion = greatestVersion, + TrackedRepositoryUrl = repoUrl, + GeneratedAt = DateTimeOffset.UtcNow, + }); } - catch (NotFoundException e) + + var ttl = TimeSpan.FromMinutes(30); + Task task; + if (fresh == true || !cacheService.TryGetValue(ReadCacheKey, out var rawCacheObject)) { - Logger.LogWarning(e, "Not found exception while retrieving upstream repository info!"); + using var entry = cacheService.CreateEntry(ReadCacheKey); + entry.AbsoluteExpirationRelativeToNow = ttl; + entry.Value = task = CacheFactory(); } + else + task = (Task)rawCacheObject!; - return Json(new AdministrationResponse - { - LatestVersion = greatestVersion, - TrackedRepositoryUrl = repoUrl, - }); + return await task; } catch (RateLimitExceededException e) { diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index d60fda967d5..79e03ebb845 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -317,11 +317,12 @@ await jobManager.RegisterOperation( /// If there was a settings change made that forced a switch to . /// The for the operation. /// A resulting in the of the operation. +#pragma warning disable CA1502 // TODO: Decomplexify ValueTask ReadImpl(DreamDaemonSettings? settings, bool knownForcedReboot, CancellationToken cancellationToken) +#pragma warning restore CA1502 => WithComponentInstance(async instance => { var dd = instance.Watchdog; - var metadata = (AuthenticationContext.GetRight(RightsType.DreamDaemon) & (ulong)DreamDaemonRights.ReadMetadata) != 0; var revision = (AuthenticationContext.GetRight(RightsType.DreamDaemon) & (ulong)DreamDaemonRights.ReadRevision) != 0; @@ -372,6 +373,7 @@ ValueTask ReadImpl(DreamDaemonSettings? settings, bool knownForce result.Visibility = settings.Visibility!.Value; result.SoftRestart = rstate == RebootState.Restart; result.SoftShutdown = rstate == RebootState.Shutdown; + result.ImmediateMemoryUsage = dd.MemoryUsage; if (rstate == RebootState.Normal && knownForcedReboot) result.SoftRestart = true; diff --git a/src/Tgstation.Server.Host/Controllers/EngineController.cs b/src/Tgstation.Server.Host/Controllers/EngineController.cs index c1a20fe99ca..83a113e3b46 100644 --- a/src/Tgstation.Server.Host/Controllers/EngineController.cs +++ b/src/Tgstation.Server.Host/Controllers/EngineController.cs @@ -184,7 +184,8 @@ public async ValueTask Update([FromBody] EngineVersionRequest mod try { - await byondManager.ChangeVersion(null, model.EngineVersion, null, false, cancellationToken); + using var progressReporter = new JobProgressReporter(); + await byondManager.ChangeVersion(progressReporter, model.EngineVersion, null, false, cancellationToken); } catch (InvalidOperationException ex) { diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 5d4d686aaf2..735b9efa839 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -223,6 +223,42 @@ await jobManager.RegisterOperation( return Accepted(api); } + /// + /// Delete the repository. + /// + /// The for the operation. + /// A resulting in the of the operation. + /// Job to delete the repository created successfully. + /// The database entity for the requested instance could not be retrieved. The instance was likely detached. + [HttpPatch] + [TgsAuthorize(RepositoryRights.Reclone)] + [ProducesResponseType(typeof(RepositoryResponse), 202)] + [ProducesResponseType(typeof(ErrorMessageResponse), 410)] + public async ValueTask Reclone(CancellationToken cancellationToken) + { + var currentModel = await DatabaseContext + .RepositorySettings + .AsQueryable() + .Where(x => x.InstanceId == Instance.Id) + .FirstOrDefaultAsync(cancellationToken); + + if (currentModel == default) + return this.Gone(); + + Logger.LogInformation("Instance {instanceId} repository reclone initiated by user {userId}", Instance.Id, AuthenticationContext.User.Require(x => x.Id)); + + var repositoryUpdater = CreateRepositoryUpdateService(currentModel); + + var job = Job.Create(JobCode.RepositoryReclone, AuthenticationContext.User, Instance); + var api = currentModel.ToApi(); + await jobManager.RegisterOperation( + job, + (core, databaseContextFactory, paramJob, progressReporter, ct) => repositoryUpdater.RepositoryRecloneJob(core, databaseContextFactory, progressReporter, ct), + cancellationToken); + api.ActiveJob = job.ToApi(); + return Accepted(api); + } + /// /// Get the repository's status. /// @@ -443,17 +479,12 @@ bool CheckModified(Expression var job = Job.Create(JobCode.RepositoryUpdate, AuthenticationContext.User, Instance, RepositoryRights.CancelPendingChanges); job.Description = description; - var repositoryUpdater = new RepositoryUpdateService( - model, - currentModel, - AuthenticationContext.User, - loggerFactory.CreateLogger(), - Instance.Require(x => x.Id)); + var repositoryUpdater = CreateRepositoryUpdateService(currentModel); // Time to access git, do it in a job await jobManager.RegisterOperation( job, - repositoryUpdater.RepositoryUpdateJob, + (instance, databaseContextFactory, _, progressReporter, jobToken) => repositoryUpdater.RepositoryUpdateJob(model, instance, databaseContextFactory, progressReporter, jobToken), cancellationToken); api.ActiveJob = job.ToApi(); @@ -494,5 +525,17 @@ async ValueTask PopulateApi( cancellationToken); return needsDbUpdate; } + + /// + /// Creates a . + /// + /// The current . + /// A new . + RepositoryUpdateService CreateRepositoryUpdateService(RepositorySettings currentModel) + => new( + currentModel, + AuthenticationContext.User, + loggerFactory.CreateLogger(), + Instance.Require(x => x.Id)); } } diff --git a/src/Tgstation.Server.Host/Controllers/RootController.cs b/src/Tgstation.Server.Host/Controllers/RootController.cs index df363970bb3..577fecf1c64 100644 --- a/src/Tgstation.Server.Host/Controllers/RootController.cs +++ b/src/Tgstation.Server.Host/Controllers/RootController.cs @@ -152,11 +152,17 @@ public IActionResult Index() [HttpGet("logo.svg")] public IActionResult GetLogo() { - var logoFileName = platformIdentifier.IsWindows // these are different because of motherfucking line endings -_- - ? LogoSvgWindowsName - : LogoSvgLinuxName; + // these are different because of motherfucking line endings -_- + if (platformIdentifier.IsWindows) + { + VirtualFileResult? result = this.TryServeFile(hostEnvironment, logger, $"{LogoSvgWindowsName}.svg"); + if (result != null) + return result; + + // BUT THE UPDATE PACKAGES ARE BUILT ON LINUX RAAAAAGH + } - return (IActionResult?)this.TryServeFile(hostEnvironment, logger, $"{logoFileName}.svg") ?? NotFound(); + return (IActionResult?)this.TryServeFile(hostEnvironment, logger, $"{LogoSvgLinuxName}.svg") ?? NotFound(); } /// diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 36970000c59..7e981066c0d 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -143,6 +143,7 @@ public void ConfigureServices( services.UseStandardConfig(Configuration); services.UseStandardConfig(Configuration); services.UseStandardConfig(Configuration); + services.UseStandardConfig(Configuration); // enable options which give us config reloading services.AddOptions(); @@ -423,6 +424,7 @@ void AddTypedContext() services.AddSingleton(); services.AddSingleton(); services.AddHostedService(); + services.AddHostedService(); services.AddFileDownloader(); services.AddGitHub(); diff --git a/src/Tgstation.Server.Host/Core/ServerUpdater.cs b/src/Tgstation.Server.Host/Core/ServerUpdater.cs index 8c9d9681687..1170537a12d 100644 --- a/src/Tgstation.Server.Host/Core/ServerUpdater.cs +++ b/src/Tgstation.Server.Host/Core/ServerUpdater.cs @@ -288,7 +288,7 @@ async ValueTask BeginUpdateImpl( { logger.LogDebug("Looking for GitHub releases version {version}...", newVersion); - var gitHubService = gitHubServiceFactory.CreateService(); + var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); var releases = await gitHubService.GetTgsReleases(cancellationToken); foreach (var kvp in releases) { diff --git a/src/Tgstation.Server.Host/Core/VersionReportingService.cs b/src/Tgstation.Server.Host/Core/VersionReportingService.cs new file mode 100644 index 00000000000..dd0eef6f786 --- /dev/null +++ b/src/Tgstation.Server.Host/Core/VersionReportingService.cs @@ -0,0 +1,269 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using Octokit; + +using Tgstation.Server.Common.Extensions; +using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Properties; +using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; + +namespace Tgstation.Server.Host.Core +{ + /// + /// Handles TGS version reporting, if enabled. + /// + sealed class VersionReportingService : BackgroundService + { + /// + /// The for the . + /// + readonly IGitHubClientFactory gitHubClientFactory; + + /// + /// The for the . + /// + readonly IIOManager ioManager; + + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + + /// + /// The for the . + /// + readonly IAssemblyInformationProvider assemblyInformationProvider; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// The for the . + /// + readonly TelemetryConfiguration telemetryConfiguration; + + /// + /// The passed to . + /// + CancellationToken shutdownCancellationToken; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + /// The value of . + /// The containing the value of . + /// The value of . + public VersionReportingService( + IGitHubClientFactory gitHubClientFactory, + IIOManager ioManager, + IAsyncDelayer asyncDelayer, + IAssemblyInformationProvider assemblyInformationProvider, + IOptions telemetryConfigurationOptions, + ILogger logger) + { + this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); + telemetryConfiguration = telemetryConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(telemetryConfigurationOptions)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public override Task StopAsync(CancellationToken cancellationToken) + { + shutdownCancellationToken = cancellationToken; + return base.StopAsync(cancellationToken); + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (telemetryConfiguration.DisableVersionReporting) + { + logger.LogDebug("Version telemetry disabled"); + return; + } + + if (!telemetryConfiguration.VersionReportingRepositoryId.HasValue) + { + logger.LogError("Version reporting repository is misconfigured. Telemetry cannot be sent!"); + return; + } + + var attribute = TelemetryAppSerializedKeyAttribute.Instance; + if (attribute == null) + { + logger.LogDebug("TGS build configuration does not allow for version telemetry"); + return; + } + + logger.LogDebug("Starting..."); + + try + { + var telemetryIdFile = ioManager.ResolvePath( + ioManager.ConcatPath( + ioManager.GetPathInLocalDirectory(assemblyInformationProvider), + "telemetry.id")); + + Guid telemetryId; + if (!await ioManager.FileExists(telemetryIdFile, stoppingToken)) + { + telemetryId = Guid.NewGuid(); + await ioManager.WriteAllBytes(telemetryIdFile, Encoding.UTF8.GetBytes(telemetryId.ToString()), stoppingToken); + logger.LogInformation("Generated telemetry ID {telemetryId} and wrote to {file}", telemetryId, telemetryIdFile); + } + else + { + var contents = await ioManager.ReadAllBytes(telemetryIdFile, stoppingToken); + + string guidStr; + try + { + guidStr = Encoding.UTF8.GetString(contents); + } + catch (Exception ex) + { + logger.LogError(ex, "Cannot decode telemetry ID from installation file ({path}). Telemetry will not be sent!", telemetryIdFile); + return; + } + + if (!Guid.TryParse(guidStr, out telemetryId)) + { + logger.LogError("Cannot parse telemetry ID from installation file ({path}). Telemetry will not be sent!", telemetryIdFile); + return; + } + } + + try + { + while (!stoppingToken.IsCancellationRequested) + { + var nextDelayHours = await TryReportVersion( + telemetryId, + attribute.SerializedKey, + telemetryConfiguration.VersionReportingRepositoryId.Value, + false, + stoppingToken) + ? 24 + : 1; + + logger.LogDebug("Next version report in {hours} hours", nextDelayHours); + await asyncDelayer.Delay(TimeSpan.FromHours(nextDelayHours), stoppingToken); + } + } + catch (OperationCanceledException ex) + { + logger.LogTrace(ex, "Inner cancellation"); + } + + shutdownCancellationToken.ThrowIfCancellationRequested(); + + logger.LogDebug("Sending shutdown telemetry"); + await TryReportVersion( + telemetryId, + attribute.SerializedKey, + telemetryConfiguration.VersionReportingRepositoryId.Value, + true, + shutdownCancellationToken); + } + catch (OperationCanceledException ex) + { + logger.LogTrace(ex, "Exiting due to outer cancellation..."); + } + catch (Exception ex) + { + logger.LogError(ex, "Crashed!"); + } + } + + /// + /// Make an attempt to report the current to the configured GitHub repository. + /// + /// The telemetry for the installation. + /// The serialized authentication for the . + /// The ID of the repository to send telemetry to. + /// If this is shutdown telemetry. + /// The for the operation. + /// A resulting in if telemetry was reported successfully, otherwise. + async ValueTask TryReportVersion(Guid telemetryId, string serializedPem, long repositoryId, bool shutdown, CancellationToken cancellationToken) + { + logger.LogDebug("Sending version telemetry..."); + + var serverFriendlyName = telemetryConfiguration.ServerFriendlyName; + if (String.IsNullOrWhiteSpace(serverFriendlyName)) + serverFriendlyName = null; + + logger.LogTrace( + "Repository ID: {repoId}, Server friendly name: {friendlyName}", + repositoryId, + serverFriendlyName == null + ? "(null)" + : $"\"{serverFriendlyName}\""); + try + { + var gitHubClient = await gitHubClientFactory.CreateInstallationClient( + serializedPem, + repositoryId, + cancellationToken); + + if (gitHubClient == null) + { + logger.LogWarning("Could not create GitHub client to connect to repository ID {repoId}!", repositoryId); + return false; + } + + // remove this lookup once https://github.com/octokit/octokit.net/pull/2960 is merged and released + var repository = await gitHubClient.Repository.Get(repositoryId); + + logger.LogTrace("Repository ID {id} resolved to {owner}/{name}", repositoryId, repository.Owner.Name, repository.Name); + + var inputs = new Dictionary + { + { "telemetry_id", telemetryId.ToString() }, + { "tgs_semver", assemblyInformationProvider.Version.Semver().ToString() }, + { "shutdown", shutdown ? "true" : "false" }, + }; + + if (serverFriendlyName != null) + inputs.Add("server_friendly_name", serverFriendlyName); + + await gitHubClient.Actions.Workflows.CreateDispatch( + repository.Owner.Login, + repository.Name, + ".github/workflows/tgs_deployments_telemetry.yml", + new CreateWorkflowDispatch("main") + { + Inputs = inputs, + }); + + logger.LogTrace("Telemetry sent successfully"); + + return true; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to report version!"); + return false; + } + } + } +} diff --git a/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs b/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs index 6d71a68edf8..b5e996c311c 100644 --- a/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs @@ -20,14 +20,14 @@ static class FetchOptionsExtensions /// /// The to hydrate. /// The for the operation. - /// The optional . + /// The . /// The optional . /// The for the operation. /// The hydrated . public static FetchOptions Hydrate( this FetchOptions fetchOptions, ILogger logger, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, CredentialsHandler credentialsHandler, CancellationToken cancellationToken) { @@ -60,10 +60,10 @@ public static FetchOptions Hydrate( /// Generate a from a given and . /// /// The for the operation. - /// The optional of the operation. + /// The of the operation. /// The for the operation. /// A new based on . - static TransferProgressHandler TransferProgressHandler(ILogger logger, JobProgressReporter? progressReporter, CancellationToken cancellationToken) => transferProgress => + static TransferProgressHandler TransferProgressHandler(ILogger logger, JobProgressReporter progressReporter, CancellationToken cancellationToken) => transferProgress => { double? percentage; var totalObjectsToProcess = transferProgress.TotalObjects * 2; diff --git a/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs b/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs index 753f8cf519e..12ee246858f 100644 --- a/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs +++ b/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs @@ -1,6 +1,7 @@ using System; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Tgstation.Server.Host.Models; @@ -9,7 +10,7 @@ namespace Tgstation.Server.Host.Jobs /// /// Progress reporter for a . /// - public sealed class JobProgressReporter + public sealed class JobProgressReporter : IDisposable { /// /// The name of the current stage. @@ -52,6 +53,24 @@ public string? StageName /// double sectionProgression; + /// + /// The total progress reserved for use in this section. + /// + double? sectionReservations; + + /// + /// Initializes a new instance of the class. + /// + /// This variant has no function. + public JobProgressReporter() + : this( + NullLogger.Instance, + null, + (_, _) => { }, + false) + { + } + /// /// Initializes a new instance of the class. /// @@ -59,27 +78,84 @@ public string? StageName /// The value of . /// The value of . public JobProgressReporter(ILogger logger, string? stageName, Action callback) + : this( + logger, + stageName, + callback, + true) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + /// If an initial call to will be made with only the . + private JobProgressReporter(ILogger logger, string? stageName, Action callback, bool setStageName) { this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.callback = callback ?? throw new ArgumentNullException(nameof(callback)); - StageName = stageName; + if (setStageName) + { + StageName = stageName; + } + else + { + this.stageName = stageName; + } logger.LogDebug("Job progress reporter created. Stage: {stageName}", stageName ?? "(null)"); } + /// + public void Dispose() + { + if (sectionReservations.HasValue) + if (sectionReservations.Value != 1.0) + { + // not an error, processes can throw + sectionReservations = null; + } + else if (sectionProgression < 1.0) + { + logger.LogError( + new InvalidOperationException($"Parent progress reporter has child sections that didn't complete! Current: {sectionProgression}"), + "TGS BUG: Progress reporter children didn't complete!"); + sectionReservations = null; + } + + if (!sectionReservations.HasValue) + ReportProgress(1); + } + /// /// Report progress. /// /// A percentage value from 0.0f-1.0f. public void ReportProgress(double? progress) { + if (sectionReservations.HasValue) + if (progress == 0) + { + // might be a stage reset + sectionReservations = null; + } + else + { + logger.LogError( + new InvalidOperationException("Progress reporter is reporting progress with existing nested sections!"), + "TGS BUG: A progress reporter is using mixed local and nested progress, this is not supported"); + } + var clampedProgress = progress; if (progress.HasValue) if (progress > 1 || progress < 0) { logger.LogError( new ArgumentOutOfRangeException(nameof(progress), progress, "Progress must be a value from 0-1!"), - "Invalid progress value for stage {stageName}", + "TGS BUG: Invalid progress value for stage {stageName}", StageName ?? "(null)"); clampedProgress = null; } @@ -103,16 +179,28 @@ public JobProgressReporter CreateSection(string? newStageName, double percentage { logger.LogError( new ArgumentOutOfRangeException(nameof(percentage), percentage, "Percentage must be a value from 0-1!"), - "Invalid percentage value for stage {newStageName}! Clamping...", + "TGS BUG: Invalid percentage value for stage {newStageName}! Clamping...", newStageName ?? "(null)"); percentage = Math.Min(Math.Max(percentage, 0.0), 1.0); } - var childBaseProgress = sectionProgression; - if (percentage + childBaseProgress > 1.0) + if (!sectionReservations.HasValue) { - var remainingPercentage = 1.0 - childBaseProgress; + if (sectionProgression != 0) + { + logger.LogError( + new InvalidOperationException("Progress reporter is creating a section with local progress!"), + "TGS BUG: A progress reporter is using mixed local and nested progress, this is not supported"); + } + + sectionReservations = 0; + } + + // floating point >.< + if (percentage + sectionReservations.Value > 1.0001) + { + var remainingPercentage = 1.0 - sectionReservations.Value; logger.LogError( "Stage {newStageName} is overbudgeted ({budget}/{remainingPercentage})! Clamping...", newStageName, @@ -121,6 +209,9 @@ public JobProgressReporter CreateSection(string? newStageName, double percentage percentage = remainingPercentage; } + Math.Min(sectionReservations.Value + percentage, 1); + + var childLocalProgress = 0.0; var newReporter = new JobProgressReporter( logger, newStageName, @@ -133,11 +224,17 @@ public JobProgressReporter CreateSection(string? newStageName, double percentage return; } - var childLocalProgress = progress.Value * percentage; + var progressWithoutChild = sectionProgression - childLocalProgress; + childLocalProgress = progress.Value * percentage; + + // floating point >.< + sectionProgression = Math.Min(progressWithoutChild + childLocalProgress, 1); + if (sectionProgression > 9.9999) + sectionProgression = 1; - sectionProgression = childLocalProgress + childBaseProgress; callback(currentStage, sectionProgression); - }); + }, + false); newReporter.ReportProgress(0); return newReporter; diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index e9967f5189c..d78cc7a2872 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -460,14 +460,16 @@ void UpdateProgress(string? stage, double? progress) QueueHubUpdate(job.ToApi(), false); logger.LogTrace("Starting job..."); + using var progressReporter = new JobProgressReporter( + loggerFactory.CreateLogger(), + null, + UpdateProgress); + using var innerReporter = progressReporter.CreateSection(null, 1.0); await operation( instanceCoreProvider.GetInstance(job.Instance!), databaseContextFactory, job, - new JobProgressReporter( - loggerFactory.CreateLogger(), - null, - UpdateProgress), + innerReporter, cancellationToken); logger.LogDebug("Job {jobId} completed!", job.Id); diff --git a/src/Tgstation.Server.Host/Properties/TelemetryAppSerializedKeyAttribute.cs b/src/Tgstation.Server.Host/Properties/TelemetryAppSerializedKeyAttribute.cs new file mode 100644 index 00000000000..6d2ff1be5b8 --- /dev/null +++ b/src/Tgstation.Server.Host/Properties/TelemetryAppSerializedKeyAttribute.cs @@ -0,0 +1,33 @@ +using System; +using System.Reflection; + +namespace Tgstation.Server.Host.Properties +{ + /// + /// Attribute for bundling the GitHub App serialized private key used for version telemetry. + /// + [AttributeUsage(AttributeTargets.Assembly)] + sealed class TelemetryAppSerializedKeyAttribute : Attribute + { + /// + /// Return the 's instance of the . + /// + public static TelemetryAppSerializedKeyAttribute? Instance => Assembly + .GetExecutingAssembly() + .GetCustomAttribute(); + + /// + /// The serialized GitHub App Client ID and private key. + /// + public string SerializedKey { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public TelemetryAppSerializedKeyAttribute(string serializedKey) + { + SerializedKey = serializedKey ?? throw new ArgumentNullException(nameof(serializedKey)); + } + } +} diff --git a/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs b/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs index 3f3180f9da2..d535d978843 100644 --- a/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs +++ b/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs @@ -60,12 +60,12 @@ public GitHubOAuthValidator( { logger.LogTrace("Validating response code..."); - var gitHubService = gitHubServiceFactory.CreateService(); + var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); var token = await gitHubService.CreateOAuthAccessToken(oAuthConfiguration, code, cancellationToken); if (token == null) return null; - var authenticatedClient = gitHubServiceFactory.CreateService(token); + var authenticatedClient = await gitHubServiceFactory.CreateService(token, cancellationToken); logger.LogTrace("Getting user details..."); var userId = await authenticatedClient.GetCurrentUserId(cancellationToken); diff --git a/src/Tgstation.Server.Host/Setup/SetupWizard.cs b/src/Tgstation.Server.Host/Setup/SetupWizard.cs index 36d0bf31116..32e82877e52 100644 --- a/src/Tgstation.Server.Host/Setup/SetupWizard.cs +++ b/src/Tgstation.Server.Host/Setup/SetupWizard.cs @@ -713,7 +713,7 @@ async ValueTask ConfigureGeneral(CancellationToken cancell while (true); await console.WriteAsync(null, true, cancellationToken); - await console.WriteAsync("Enter a GitHub personal access token to bypass some rate limits (this is optional and does not require any scopes)", true, cancellationToken); + await console.WriteAsync("Enter a classic GitHub personal access token to bypass some rate limits (this is optional and does not require any scopes)", true, cancellationToken); await console.WriteAsync("GitHub personal access token: ", false, cancellationToken); newGeneralConfiguration.GitHubAccessToken = await console.ReadLineAsync(true, cancellationToken); if (String.IsNullOrWhiteSpace(newGeneralConfiguration.GitHubAccessToken)) @@ -958,6 +958,31 @@ async ValueTask ParseAddress(string question) }; } + /// + /// Prompts the user to create a . + /// + /// The for the operation. + /// A resulting in the new . + async ValueTask ConfigureTelemetry(CancellationToken cancellationToken) + { + bool enableReporting = await PromptYesNo("Enable version telemetry? This anonymously reports the TGS version in use.", true, cancellationToken); + + string? serverFriendlyName = null; + if (enableReporting) + { + await console.WriteAsync("(Optional) Publically associate your reported version with a friendly name:", false, cancellationToken); + serverFriendlyName = await console.ReadLineAsync(false, cancellationToken); + if (String.IsNullOrWhiteSpace(serverFriendlyName)) + serverFriendlyName = null; + } + + return new TelemetryConfiguration + { + DisableVersionReporting = !enableReporting, + ServerFriendlyName = serverFriendlyName, + }; + } + /// /// Saves a given set to . /// @@ -969,6 +994,7 @@ async ValueTask ParseAddress(string question) /// The to save. /// The to save. /// The to save. + /// The to save. /// The for the operation. /// A representing the running operation. async ValueTask SaveConfiguration( @@ -980,6 +1006,7 @@ async ValueTask SaveConfiguration( ElasticsearchConfiguration? elasticsearchConfiguration, ControlPanelConfiguration controlPanelConfiguration, SwarmConfiguration? swarmConfiguration, + TelemetryConfiguration? telemetryConfiguration, CancellationToken cancellationToken) { newGeneralConfiguration.ApiPort = hostingPort ?? GeneralConfiguration.DefaultApiPort; @@ -992,6 +1019,7 @@ async ValueTask SaveConfiguration( { ElasticsearchConfiguration.Section, elasticsearchConfiguration }, { ControlPanelConfiguration.Section, controlPanelConfiguration }, { SwarmConfiguration.Section, swarmConfiguration }, + { TelemetryConfiguration.Section, telemetryConfiguration }, }; var versionConverter = new VersionConverter(); @@ -1065,6 +1093,8 @@ async ValueTask RunWizard(string userConfigFileName, CancellationToken cancellat var swarmConfiguration = await ConfigureSwarm(cancellationToken); + var telemetryConfiguration = await ConfigureTelemetry(cancellationToken); + await console.WriteAsync(null, true, cancellationToken); await console.WriteAsync(String.Format(CultureInfo.InvariantCulture, "Configuration complete! Saving to {0}", userConfigFileName), true, cancellationToken); @@ -1077,6 +1107,7 @@ await SaveConfiguration( elasticSearchConfiguration, controlPanelConfiguration, swarmConfiguration, + telemetryConfiguration, cancellationToken); } @@ -1194,6 +1225,7 @@ await SaveConfiguration( AllowAnyOrigin = true, }, null, + null, cancellationToken); } else diff --git a/src/Tgstation.Server.Host/System/IProcessBase.cs b/src/Tgstation.Server.Host/System/IProcessBase.cs index 92f6bdb1590..6ae88dce502 100644 --- a/src/Tgstation.Server.Host/System/IProcessBase.cs +++ b/src/Tgstation.Server.Host/System/IProcessBase.cs @@ -13,6 +13,11 @@ public interface IProcessBase /// Task Lifetime { get; } + /// + /// Gets the process' memory usage in bytes. + /// + long? MemoryUsage { get; } + /// /// Set's the owned to a non-normal value. /// diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index 896195944c5..13a1a278f50 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -22,6 +22,23 @@ sealed class Process : IProcess /// public Task Lifetime { get; } + /// + public long? MemoryUsage + { + get + { + try + { + return handle.VirtualMemorySize64; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to get PID {pid}'s memory usage!", Id); + return null; + } + } + } + /// /// The for the . /// diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index cf412dc3ce5..8fc7dca66bb 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -29,13 +29,13 @@ - + - + @@ -45,21 +45,43 @@ - + <_Parameter1>$(TgsConfigVersion) <_Parameter2>$(TgsInteropVersion) <_Parameter3>$(TgsWebpanelVersion) <_Parameter4>$(TgsHostWatchdogVersion) <_Parameter5>$(TgsMariaDBRedistVersion) - + + + + + + - + + + + + + + + <_Parameter1>@(SerializedTelemetryKey) + + + + + + + @@ -78,21 +100,21 @@ - + - + - + - + runtime; build; native; contentfiles; analyzers; buildtransitive - + - + @@ -115,9 +137,9 @@ - + - + diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs index e479388b79b..4cf31a5d289 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs @@ -1,9 +1,16 @@ using System; using System.Collections.Generic; +using System.IdentityModel.Tokens.Jwt; using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; + using Octokit; using Tgstation.Server.Host.Configuration; @@ -12,7 +19,7 @@ namespace Tgstation.Server.Host.Utils.GitHub { /// - sealed class GitHubClientFactory : IGitHubClientFactory + sealed class GitHubClientFactory : IGitHubClientFactory, IDisposable { /// /// Limit to the amount of days a can live in the . @@ -45,6 +52,11 @@ sealed class GitHubClientFactory : IGitHubClientFactory /// readonly Dictionary clientCache; + /// + /// The used to guard access to . + /// + readonly SemaphoreSlim clientCacheSemaphore; + /// /// Initializes a new instance of the class. /// @@ -61,50 +73,139 @@ public GitHubClientFactory( generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); clientCache = new Dictionary(); + clientCacheSemaphore = new SemaphoreSlim(1, 1); } /// - public IGitHubClient CreateClient() => GetOrCreateClient(generalConfiguration.GitHubAccessToken); + public void Dispose() => clientCacheSemaphore.Dispose(); + + /// + public async ValueTask CreateClient(CancellationToken cancellationToken) + => (await GetOrCreateClient( + generalConfiguration.GitHubAccessToken, + null, + cancellationToken))!; /// - public IGitHubClient CreateClient(string accessToken) - => GetOrCreateClient( - accessToken ?? throw new ArgumentNullException(nameof(accessToken))); + public async ValueTask CreateClient(string accessToken, CancellationToken cancellationToken) + => (await GetOrCreateClient( + accessToken ?? throw new ArgumentNullException(nameof(accessToken)), + null, + cancellationToken))!; + + /// + public ValueTask CreateInstallationClient(string serializedPem, long repositoryId, CancellationToken cancellationToken) + => GetOrCreateClient(serializedPem, repositoryId, cancellationToken); /// - /// Retrieve a from the or add a new one based on a given . + /// Retrieve a from the or add a new one based on a given . /// - /// Optional access token to use as credentials. - /// The for the given . - GitHubClient GetOrCreateClient(string? accessToken) + /// Optional access token to use as credentials or GitHub App private key. If using a private key, must be set. + /// Setting this specifies is a private key and a GitHub App installation authenticated client will be returned. + /// The for the operation. + /// A resulting in the for the given or if authentication failed. +#pragma warning disable CA1506 // TODO: Decomplexify + async ValueTask GetOrCreateClient(string? accessTokenOrSerializedPem, long? installationRepositoryId, CancellationToken cancellationToken) +#pragma warning restore CA1506 { GitHubClient client; bool cacheHit; DateTimeOffset? lastUsed; - lock (clientCache) + using (await SemaphoreSlimContext.Lock(clientCacheSemaphore, cancellationToken)) { string cacheKey; - if (String.IsNullOrWhiteSpace(accessToken)) + if (String.IsNullOrWhiteSpace(accessTokenOrSerializedPem)) { - accessToken = null; + accessTokenOrSerializedPem = null; cacheKey = DefaultCacheKey; } else - cacheKey = accessToken; + cacheKey = accessTokenOrSerializedPem; cacheHit = clientCache.TryGetValue(cacheKey, out var tuple); var now = DateTimeOffset.UtcNow; if (!cacheHit) { + logger.LogTrace("Creating new GitHubClient..."); var product = assemblyInformationProvider.ProductInfoHeaderValue.Product!; client = new GitHubClient( new ProductHeaderValue( product.Name, product.Version)); - if (accessToken != null) - client.Credentials = new Credentials(accessToken); + if (accessTokenOrSerializedPem != null) + { + if (installationRepositoryId.HasValue) + { + logger.LogTrace("Performing GitHub App authentication for installation on repository {installationRepositoryId}", installationRepositoryId.Value); + var splits = accessTokenOrSerializedPem.Split(':'); + if (splits.Length != 2) + { + logger.LogError("Failed to parse serialized Client ID & PEM! Expected 2 chunks, got {chunkCount}", splits.Length); + return null; + } + + byte[] pemBytes; + try + { + pemBytes = Convert.FromBase64String(splits[1]); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to parse supposed base64 PEM!"); + return null; + } + + var pem = Encoding.UTF8.GetString(pemBytes); + + using var rsa = RSA.Create(); + rsa.ImportFromPem(pem); + + var signingCredentials = new SigningCredentials(new RsaSecurityKey(rsa), SecurityAlgorithms.RsaSha256); + var jwtSecurityTokenHandler = new JwtSecurityTokenHandler { SetDefaultTimesOnTokenCreation = false }; + + var nowDateTime = DateTime.UtcNow; + + var jwt = jwtSecurityTokenHandler.CreateToken(new SecurityTokenDescriptor + { + Issuer = splits[0], + Expires = nowDateTime.AddMinutes(10), + IssuedAt = nowDateTime, + SigningCredentials = signingCredentials, + }); + + var jwtStr = jwtSecurityTokenHandler.WriteToken(jwt); + + client.Credentials = new Credentials(jwtStr, AuthenticationType.Bearer); + + Installation installation; + try + { + installation = await client.GitHubApps.GetRepositoryInstallationForCurrent(installationRepositoryId.Value); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to perform app authentication!"); + return null; + } + + cancellationToken.ThrowIfCancellationRequested(); + try + { + var installToken = await client.GitHubApps.CreateInstallationToken(installation.Id); + + client.Credentials = new Credentials(installToken.Token); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to perform installation authentication!"); + return null; + } + } + else + client.Credentials = new Credentials(accessTokenOrSerializedPem); + } clientCache.Add(cacheKey, (Client: client, LastUsed: now)); lastUsed = null; diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs index efa08d13bba..a38b84b271d 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs @@ -1,4 +1,6 @@ using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -44,13 +46,16 @@ public GitHubServiceFactory( } /// - public IGitHubService CreateService() => CreateServiceImpl(gitHubClientFactory.CreateClient()); + public async ValueTask CreateService(CancellationToken cancellationToken) + => CreateServiceImpl( + await gitHubClientFactory.CreateClient(cancellationToken)); /// - public IAuthenticatedGitHubService CreateService(string accessToken) + public async ValueTask CreateService(string accessToken, CancellationToken cancellationToken) => CreateServiceImpl( - gitHubClientFactory.CreateClient( - accessToken ?? throw new ArgumentNullException(nameof(accessToken)))); + await gitHubClientFactory.CreateClient( + accessToken ?? throw new ArgumentNullException(nameof(accessToken)), + cancellationToken)); /// /// Create a . diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs index ec95fd16ab8..1808b4ca3c3 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs @@ -1,4 +1,7 @@ -using Octokit; +using System.Threading; +using System.Threading.Tasks; + +using Octokit; namespace Tgstation.Server.Host.Utils.GitHub { @@ -10,14 +13,25 @@ public interface IGitHubClientFactory /// /// Create a client. Low rate limit unless the server's GitHubAccessToken is set to bypass it. /// + /// The for the operation. /// A new . - IGitHubClient CreateClient(); + ValueTask CreateClient(CancellationToken cancellationToken); /// /// Create a client with authentication using a personal access token. /// /// The GitHub personal access token. + /// The for the operation. /// A new . - IGitHubClient CreateClient(string accessToken); + ValueTask CreateClient(string accessToken, CancellationToken cancellationToken); + + /// + /// Creates a GitHub App client for an installation. + /// + /// The private key . + /// The GitHub repository ID. + /// The for the operation. + /// A resulting in a new for the given or if authentication failed. + ValueTask CreateInstallationClient(string pem, long repositoryId, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs index f4e7be70f55..adb0ec88ebd 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs @@ -1,4 +1,7 @@ -namespace Tgstation.Server.Host.Utils.GitHub +using System.Threading; +using System.Threading.Tasks; + +namespace Tgstation.Server.Host.Utils.GitHub { /// /// Factory for s. @@ -8,14 +11,16 @@ public interface IGitHubServiceFactory /// /// Create a . /// - /// A new . - public IGitHubService CreateService(); + /// The for the operation. + /// A resulting in a new . + public ValueTask CreateService(CancellationToken cancellationToken); /// /// Create an . /// /// The access token to use for communication with GitHub. - /// A new . - public IAuthenticatedGitHubService CreateService(string accessToken); + /// The for the operation. + /// A resulting in a new . + public ValueTask CreateService(string accessToken, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs index a700ccc9cba..dc15d9db687 100644 --- a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs +++ b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs @@ -76,7 +76,7 @@ public static void Configure(SwaggerGenOptions swaggerGenOptions, string assembl Name = "/tg/station 13", Url = new Uri("https://github.com/tgstation"), }, - Description = "A production scale tool for BYOND server management", + Description = "A production scale tool for DreamMaker server management", }); // Important to do this before applying our own filters diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 4868f519fd3..353104e6de7 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -3,7 +3,7 @@ General: # ConfigVersion: # Basic semver. Differs from TGS version to version. See changelog for current version MinimumPasswordLength: 15 # Minimum TGS user password length - GitHubAccessToken: # GitHub personal access token with no scopes used to bypass rate-limits + GitHubAccessToken: # A classic GitHub personal access token with no scopes used to bypass rate-limits SetupWizardMode: AutoDetect # If the interactive TGS setup wizard should run ByondTopicTimeout: 5000 # Timeout for BYOND /world/Topic() calls in milliseconds RestartTimeoutMinutes: 1 # Timeout for server restarts after requested by SIGTERM or the HTTP API @@ -23,6 +23,7 @@ Session: HighPriorityLiveDreamDaemon: false # If DreamDaemon instances should run as higher priority processes LowPriorityDeploymentProcesses: true # If TGS Deployments should run as lower priority processes DelayCleaningFailedDeployments: false # If true, deployments that fail will not be immediately cleaned up. They will be cleaned up the next time the instance is onlined + ForceUseDreamDaemonExe: false # If true, dd.exe will not be used on Windows systems in versions where it is present. Instead dreamdaemon.exe will always be used. FileLogging: Directory: # Directory in which log files are stored. Windows default: %PROGRAMDATA%/tgstation-server. Linux default: /var/log/tgstation-server Disable: true # Disable file logging entirely @@ -77,3 +78,7 @@ Swarm: # Should be left empty if using swarm mode is not desired # PublicAddress: # The public address of the swarm node # ControllerAddress: # Required on non-controller nodes. The internal address of the swarm controller's API'. Should be left empty on the controller itself # UpdateRequiredNodeCount: # The number of nodes expected to be in the swarm before initiating an update. This should count every server irrespective of whether or not they are the controller MINUS 1 +Telemetry: + DisableVersionReporting: false # Prevents you installation and the version you're using from being reported on the source repository's deployments list + ServerFriendlyName: null # Sets a friendly name for your server in reported telemetry. Must be unique. First come first serve + VersionReportingRepositoryId: 841149827 # GitHub repostiory ID where the tgs_version_telemetry workflow can be found diff --git a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj index 01518a39b01..c092a103f7f 100644 --- a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj +++ b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj @@ -10,7 +10,7 @@ - + diff --git a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs index 2a62fea29ad..8950acc6ab6 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs @@ -13,6 +13,7 @@ using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; @@ -52,7 +53,7 @@ static async Task RepoDownloadTest(bool needsClone) null, null, null, - null, + It.IsNotNull(), true, It.IsAny())) .Callback(() => ++cloneAttempts) @@ -85,7 +86,7 @@ static async Task RepoDownloadTest(bool needsClone) Engine = EngineType.OpenDream, SourceSHA = new string('a', Limits.MaximumCommitShaLength), }, - null, + new JobProgressReporter(), CancellationToken.None); diff --git a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs index c4a17f51401..2cdd59d19b8 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs @@ -15,6 +15,7 @@ using Tgstation.Server.Host.Components.Events; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; namespace Tgstation.Server.Host.Components.Repository.Tests { @@ -86,7 +87,7 @@ public async Task TestBasicClone() null, null, null, - null, + new JobProgressReporter(), false, CancellationToken.None); diff --git a/tests/Tgstation.Server.Host.Tests/Jobs/TestJobProgressReporter.cs b/tests/Tgstation.Server.Host.Tests/Jobs/TestJobProgressReporter.cs new file mode 100644 index 00000000000..f6d6cd9348f --- /dev/null +++ b/tests/Tgstation.Server.Host.Tests/Jobs/TestJobProgressReporter.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using Moq; + +namespace Tgstation.Server.Host.Jobs.Tests +{ + [TestClass] + public sealed class TestJobProgressReporter + { + string expectedStageName = null; + double? expectedProgress = null; + void Validate(string stageName, double? progress) + { + Assert.AreEqual(expectedStageName, stageName); + Assert.AreEqual(expectedProgress, progress); + } + + JobProgressReporter Setup() + { + expectedStageName = null; + expectedProgress = 0; + return new JobProgressReporter( + Mock.Of>(), + null, + Validate); + } + + [TestMethod] + public void TestBasicUsage() + { + var progressReporter = Setup(); + + expectedProgress = 0.4; + progressReporter.ReportProgress(0.4); + expectedProgress = 1.0; + progressReporter.ReportProgress(1.0); + } + + [TestMethod] + public void TestNestedUsage() + { + var progressReporter = Setup(); + + expectedStageName = "Test1"; + var subReporter1 = progressReporter.CreateSection("Test1", 0.5); + expectedProgress = 0.1; + subReporter1.ReportProgress(0.2); + expectedProgress = 0.4; + subReporter1.ReportProgress(0.8); + + expectedStageName = "Test2"; + var subReporter2 = progressReporter.CreateSection("Test2", 0.5); + + expectedStageName = "Test1"; + expectedProgress = 0.5; + subReporter1.ReportProgress(1); + + expectedStageName = "Test2"; + expectedProgress = 0.6; + subReporter2.ReportProgress(0.2); + expectedProgress = 1.0; + subReporter2.ReportProgress(1); + } + } +} diff --git a/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs b/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs index 1aa35762774..5598673dc29 100644 --- a/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs +++ b/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs @@ -195,6 +195,8 @@ public async Task TestWithUserStupidity() "y", // swarm config "n", + // telemetry config + "n", //saved, now for second run //this time use defaults amap String.Empty, @@ -234,6 +236,9 @@ public async Task TestWithUserStupidity() "privatekey", "n", "http://controller.com", + // telemetry config + "y", + "telemetry name", //third run, we already hit all the code coverage so just get through it String.Empty, nameof(DatabaseType.MariaDB), @@ -271,7 +276,9 @@ public async Task TestWithUserStupidity() "https://controllerinternal.com", "https://controllerpublic.com", "privatekey", - "y" + "y", + // telemetry config + "n", }; var inputPos = 0; diff --git a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj index 5e2a7f71e96..261c136fc7b 100644 --- a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj +++ b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs index 6f4fac0ae26..33058be75f4 100644 --- a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs @@ -1,5 +1,6 @@ using System; using System.Net.Http.Headers; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -57,14 +58,14 @@ public async Task TestCreateBasicClient() mockOptions.SetupGet(x => x.Value).Returns(gc); var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); - var client = factory.CreateClient(); + var client = await factory.CreateClient(CancellationToken.None); Assert.IsNotNull(client); var credentials = await client.Connection.CredentialStore.GetCredentials(); Assert.AreEqual(AuthenticationType.Anonymous, credentials.AuthenticationType); gc.GitHubAccessToken = "asdfasdfasdfasdfasdfasdf"; - client = factory.CreateClient(); + client = await factory.CreateClient(CancellationToken.None); Assert.IsNotNull(client); credentials = await client.Connection.CredentialStore.GetCredentials(); @@ -83,9 +84,9 @@ public async Task TestCreateTokenClient() mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); - Assert.ThrowsException(() => factory.CreateClient(null)); + await Assert.ThrowsExceptionAsync(() => factory.CreateClient(null, CancellationToken.None).AsTask()); - var client = factory.CreateClient("asdf"); + var client = await factory.CreateClient("asdf", CancellationToken.None); Assert.IsNotNull(client); var credentials = await client.Connection.CredentialStore.GetCredentials(); @@ -96,7 +97,7 @@ public async Task TestCreateTokenClient() } [TestMethod] - public void TestClientCaching() + public async Task TestClientCaching() { var mockApp = new Mock(); mockApp.SetupGet(x => x.ProductInfoHeaderValue).Returns(new ProductInfoHeaderValue("TGSTests", "1.2.3")).Verifiable(); @@ -105,10 +106,10 @@ public void TestClientCaching() mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); - var client1 = factory.CreateClient(); - var client2 = factory.CreateClient("asdf"); - var client3 = factory.CreateClient(); - var client4 = factory.CreateClient("asdf"); + var client1 = await factory.CreateClient(CancellationToken.None); + var client2 = await factory.CreateClient("asdf", CancellationToken.None); + var client3 = await factory.CreateClient(CancellationToken.None); + var client4 = await factory.CreateClient("asdf", CancellationToken.None); Assert.ReferenceEquals(client1, client3); Assert.ReferenceEquals(client2, client4); } diff --git a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs index daa9afdd4c2..24212eaa936 100644 --- a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs @@ -1,4 +1,6 @@ using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -28,27 +30,29 @@ public void TestConstructor() } [TestMethod] - public void TestCreateService() + public async Task TestCreateService() { var mockFactory = new Mock(); - mockFactory.Setup(x => x.CreateClient()).Returns(Mock.Of()).Verifiable(); +#pragma warning disable CA2012 // Use ValueTasks correctly + mockFactory.Setup(x => x.CreateClient(It.IsAny())).Returns(ValueTask.FromResult(Mock.Of())).Verifiable(); var mockToken = "asdf"; - mockFactory.Setup(x => x.CreateClient(mockToken)).Returns(Mock.Of()).Verifiable(); + mockFactory.Setup(x => x.CreateClient(mockToken, It.IsAny())).Returns(ValueTask.FromResult(Mock.Of())).Verifiable(); +#pragma warning restore CA2012 // Use ValueTasks correctly var mockOptions = new Mock>(); mockOptions.SetupGet(x => x.Value).Returns(new UpdatesConfiguration()); var factory = new GitHubServiceFactory(mockFactory.Object, Mock.Of(), mockOptions.Object); - Assert.ThrowsException(() => factory.CreateService(null)); + await Assert.ThrowsExceptionAsync(() => factory.CreateService(null, CancellationToken.None).AsTask()); Assert.AreEqual(0, mockFactory.Invocations.Count); - var result1 = factory.CreateService(); + var result1 = await factory.CreateService(CancellationToken.None); Assert.IsNotNull(result1); - var result2 = factory.CreateService(mockToken); + var result2 = factory.CreateService(mockToken, CancellationToken.None); Assert.IsNotNull(result2); mockFactory.VerifyAll(); diff --git a/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs b/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs index adfb009d697..afeabef6f5d 100644 --- a/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs @@ -1,10 +1,10 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; +using System; using System.IO; -using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Client; @@ -54,10 +54,24 @@ await ApiAssert.ThrowsException CreateDummyService(); + public ValueTask CreateService(CancellationToken cancellationToken) => ValueTask.FromResult(CreateDummyService()); - public IAuthenticatedGitHubService CreateService(string accessToken) + public ValueTask CreateService(string accessToken, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(accessToken); - return CreateDummyService(); + return ValueTask.FromResult(CreateDummyService()); } TestingGitHubService CreateDummyService() => new TestingGitHubService(cryptographySuite, logger); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 9c2b44c9624..67ed9e2178f 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -22,6 +22,7 @@ using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; @@ -142,7 +143,7 @@ public static async ValueTask DownloadEngineVersion( using var windowsByondInstaller = byondInstaller as WindowsByondInstaller; // get the bytes for stable - return await byondInstaller.DownloadVersion(compatVersion, null, cancellationToken); + return await byondInstaller.DownloadVersion(compatVersion, new JobProgressReporter(), cancellationToken); } public async Task RunCompatTests( diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index b3149556e56..70b7e9c030f 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -262,7 +262,7 @@ public async ValueTask WaitForReconnect(CancellationToken cancellationToken) Assert.AreEqual(HubConnectionState.Disconnected, permlessConn.State); // force token refreshs - await Task.WhenAll(permedUser.Administration.Read(cancellationToken).AsTask(), permlessUser.Instances.List(null, cancellationToken).AsTask()); + await Task.WhenAll(permedUser.Administration.Read(false, cancellationToken).AsTask(), permlessUser.Instances.List(null, cancellationToken).AsTask()); if (!permlessPsId.HasValue) { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs index 0d6ecce8648..86fe4ac9119 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs @@ -116,6 +116,8 @@ public async Task AbortLongCloneAndCloneSomethingQuick(Task longClo await ApiAssert.ThrowsException(() => Checkout(new RepositoryUpdateRequest { Reference = "master", CheckoutSha = "286bb75" }, false, false, cancellationToken), ErrorCode.RepoMismatchShaAndReference); var updated = await Checkout(new RepositoryUpdateRequest { CheckoutSha = "286bb75" }, false, false, cancellationToken); + await RecloneTest(cancellationToken); + // Fake SHA updated = await Checkout(new RepositoryUpdateRequest { CheckoutSha = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, true, false, cancellationToken); @@ -142,6 +144,23 @@ await repositoryClient.Update(new RepositoryUpdateRequest await TestMergeTests(updated, prNumber, cancellationToken); } + async ValueTask RecloneTest(CancellationToken cancellationToken) + { + var initialState = await repositoryClient.Read(cancellationToken); + Assert.IsNotNull(initialState.Reference); + Assert.IsNotNull(initialState.RevisionInformation); + Assert.IsNotNull(initialState.RevisionInformation.CommitSha); + Assert.IsNotNull(initialState.RevisionInformation.OriginCommitSha); + + var reclone = await repositoryClient.Reclone(cancellationToken); + await WaitForJob(reclone.ActiveJob, 70, false, null, cancellationToken); + + var newState = await repositoryClient.Read(cancellationToken); + Assert.AreEqual(initialState.Reference, newState.Reference); + Assert.AreEqual(initialState.RevisionInformation.CommitSha, newState.RevisionInformation.CommitSha); + Assert.AreEqual(initialState.RevisionInformation.OriginCommitSha, newState.RevisionInformation.OriginCommitSha); + } + async ValueTask Checkout(RepositoryUpdateRequest updated, bool expectFailure, bool isRef, CancellationToken cancellationToken) { var newRef = isRef ? updated.Reference : updated.CheckoutSha; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 26bba79305f..08f436b26df 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -150,6 +150,7 @@ async Task UpdateDDSettings() }, cancellationToken); Assert.AreEqual(47, updated.OpenDreamTopicPort); + Assert.IsFalse(updated.ImmediateMemoryUsage.HasValue); } catch (ConflictException ex) when (ex.ErrorCode == ErrorCode.PortNotAvailable) { @@ -721,6 +722,8 @@ async Task RunBasicTest(CancellationToken cancellationToken) await CheckDDPriority(); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); + Assert.IsTrue(daemonStatus.ImmediateMemoryUsage.HasValue); + Assert.AreNotEqual(0, daemonStatus.ImmediateMemoryUsage.Value); await GracefulWatchdogShutdown(cancellationToken); @@ -1365,6 +1368,9 @@ async Task RunLongRunningTestThenUpdateWithByondVersionSwitch(CancellationToken Assert.IsNotNull(daemonStatus.ActiveCompileJob); ValidateSessionId(daemonStatus, true); + Assert.IsTrue(daemonStatus.ImmediateMemoryUsage.HasValue); + Assert.AreNotEqual(0, daemonStatus.ImmediateMemoryUsage.Value); + Assert.AreEqual(initialStatus.ActiveCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; Assert.AreNotEqual(daemonStatus.ActiveCompileJob.EngineVersion, newerCompileJob.EngineVersion); diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 645edac147d..f9ed0b926ab 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -155,6 +155,7 @@ public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth $"General:OpenDreamGitUrl={OpenDreamUrl}", $"Security:TokenExpiryMinutes=120", // timeouts are useless for us $"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}", + "Telemetry:DisableVersionReporting=true", }; swarmArgs = new List(); diff --git a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs index 3e20609e600..30e6b9e78e9 100644 --- a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs +++ b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs @@ -216,7 +216,7 @@ static async Task TestServerInformation(IServerClientFactory clientFactory, ISer }; var badClient = clientFactory.CreateFromToken(serverClient.Url, newToken); - await ApiAssert.ThrowsException(() => badClient.Administration.Read(cancellationToken)); + await ApiAssert.ThrowsException(() => badClient.Administration.Read(false, cancellationToken)); await ApiAssert.ThrowsException(() => badClient.ServerInformation(cancellationToken)); } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index bfc4538c685..b34b14f066d 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -670,7 +670,7 @@ await Task.WhenAny( "asdfasdfasdfasdf"); await using var node1BadClient = clientFactory.CreateFromToken(node1.RootUrl, controllerUserClient.Token); - await ApiAssert.ThrowsException(() => node1BadClient.Administration.Read(cancellationToken)); + await ApiAssert.ThrowsException(() => node1BadClient.Administration.Read(false, cancellationToken)); // check instance info is not shared var controllerInstance = await controllerClient.Instances.CreateOrAttach( diff --git a/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs b/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs index d104dbd6d3a..585dd569997 100644 --- a/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs +++ b/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs @@ -39,7 +39,7 @@ static TestingGitHubService() }); var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), Mock.Of>(), mockOptions.Object); - RealClient = gitHubClientFactory.CreateClient(); + RealClient = gitHubClientFactory.CreateClient(CancellationToken.None).GetAwaiter().GetResult(); } public static async Task InitializeAndInject(CancellationToken cancellationToken) diff --git a/tests/Tgstation.Server.Tests/TestRepository.cs b/tests/Tgstation.Server.Tests/TestRepository.cs index e73731de404..59b47839942 100644 --- a/tests/Tgstation.Server.Tests/TestRepository.cs +++ b/tests/Tgstation.Server.Tests/TestRepository.cs @@ -45,7 +45,7 @@ public async Task TestRepoParentLookup() () => { }); const string StartSha = "af4da8beb9f9b374b04a3cc4d65acca662e8cc1a"; - await repo.CheckoutObject(StartSha, null, null, true, new JobProgressReporter(Mock.Of>(), null, (stage, progress) => { }), CancellationToken.None); + await repo.CheckoutObject(StartSha, null, null, true, false, new JobProgressReporter(Mock.Of>(), null, (stage, progress) => { }), CancellationToken.None); Assert.AreEqual(Host.Components.Repository.Repository.NoReference, repo.Reference); diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index b3856534865..c59f86fd19b 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1,7 +1,6 @@ // This program is minimal effort and should be sent to remedial school using System; -using System.Buffers.Text; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; @@ -60,8 +59,8 @@ static async Task Main(string[] args) var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase); - var ciCompletionCheck = versionString.Equals("--ci-completion-check", StringComparison.OrdinalIgnoreCase); var genToken = versionString.Equals("--token-output-file", StringComparison.OrdinalIgnoreCase); + var waitCodecov = versionString.Equals("--wait-codecov", StringComparison.OrdinalIgnoreCase); if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease @@ -69,8 +68,8 @@ static async Task Main(string[] args) && !shaCheck && !fullNotes && !nuget - && !ciCompletionCheck - && !genToken) + && !genToken + && !waitCodecov) { Console.WriteLine("Invalid version: " + versionString); return 2; @@ -111,20 +110,7 @@ static async Task Main(string[] args) break; } - const string ReleaseNotesEnvVar = "TGS_RELEASE_NOTES_TOKEN"; - var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar); - if (String.IsNullOrWhiteSpace(githubToken) && !doNotCloseMilestone && !ensureRelease) - { - Console.WriteLine("Missing " + ReleaseNotesEnvVar + " environment variable!"); - return 3; - } - var client = new GitHubClient(new Octokit.ProductHeaderValue("tgs_release_notes")); - if (!String.IsNullOrWhiteSpace(githubToken)) - { - client.Credentials = new Credentials(githubToken); - } - try { if (ensureRelease) @@ -135,54 +121,61 @@ static async Task Main(string[] args) return 454233; } - await GenerateAppCredentials(client, args[1]); + await GenerateAppCredentials(client, args[1], false); return await EnsureRelease(client); } - if (linkWinget) + if (genToken) { - if (args.Length < 2 || !Uri.TryCreate(args[1], new UriCreationOptions(), out var actionsUrl)) + if (args.Length < 3) { - Console.WriteLine("Missing/Invalid actions URL!"); - return 30; + Console.WriteLine("Missing output file path or PEM Base64 for app authentication!"); + return 33847; } - return await Winget(client, actionsUrl, null); + bool toSS13 = args.Length > 3 && args[3].Equals("--spacestation13", StringComparison.OrdinalIgnoreCase); + await GenerateAppCredentials(client, args[2], toSS13); + + var token = client.Credentials.GetToken(); + var destPath = args[1]; + Directory.CreateDirectory(Path.GetDirectoryName(destPath)); + await File.WriteAllTextAsync(destPath, token); + return 0; } - if (ciCompletionCheck) + const string ReleaseNotesEnvVar = "TGS_RELEASE_NOTES_TOKEN"; + var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar); + if (String.IsNullOrWhiteSpace(githubToken) && !doNotCloseMilestone && !ensureRelease) { - if (args.Length < 3) - { - Console.WriteLine("Missing SHA or PEM Base64 for creating check run!"); - return 4543; - } + Console.WriteLine("Missing " + ReleaseNotesEnvVar + " environment variable!"); + return 3; + } - return await CICompletionCheck(client, args[1], args[2]); + if (!String.IsNullOrWhiteSpace(githubToken)) + { + client.Credentials = new Credentials(githubToken); } + if (waitCodecov) + { + return await CodecovCheck(client, Int64.Parse(args[1])); + } - if (genToken) + if (linkWinget) { - if (args.Length < 3) + if (args.Length < 2 || !Uri.TryCreate(args[1], new UriCreationOptions(), out var actionsUrl)) { - Console.WriteLine("Missing output file path or PEM Base64 for app authentication!"); - return 33847; + Console.WriteLine("Missing/Invalid actions URL!"); + return 30; } - await GenerateAppCredentials(client, args[2]); - - var token = client.Credentials.GetToken(); - var destPath = args[1]; - Directory.CreateDirectory(Path.GetDirectoryName(destPath)); - await File.WriteAllTextAsync(destPath, token); - return 0; + return await Winget(client, actionsUrl, null); } if (shaCheck) { - if(args.Length < 2) + if (args.Length < 2) { Console.WriteLine("Missing SHA for PR template!"); return 32; @@ -764,7 +757,7 @@ async Task CommitNotes(Component component, List notes) } if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal) || trimmedLine.StartsWith("/🆑", StringComparison.Ordinal)) { - if(!Enum.TryParse(targetComponent, out var component)) + if (!Enum.TryParse(targetComponent, out var component)) component = targetComponent.ToUpperInvariant() switch { "**CONFIGURATION**" or "CONFIGURATION" or "CONFIG" => Component.Configuration, @@ -890,7 +883,7 @@ The user account that created this pull request is available to correct any issu }); var prToModify = userPrsOnWingetRepo.Items.OrderByDescending(pr => pr.Number).FirstOrDefault(); - if(prToModify == null) + if (prToModify == null) { Console.WriteLine("Could not find open winget-pkgs PR!"); return 31; @@ -1013,7 +1006,7 @@ async Task RunPRs() .GroupBy(kvp => kvp.Key) .Select(grouping => new KeyValuePair(grouping.Key, grouping.Max(kvp => kvp.Value)))); - foreach(var maxVersionKvp in prResults.SelectMany(x => x.Item1) + foreach (var maxVersionKvp in prResults.SelectMany(x => x.Item1) .Where(x => !releasedComponentVersions.ContainsKey(x.Key)) .GroupBy(x => x.Key) .Select(group => { @@ -1039,7 +1032,7 @@ async Task RunPRs() var component = componentKvp.Key; var list = new List(); - foreach(var changelistDict in prResults.Select(x => x.Item1)) + foreach (var changelistDict in prResults.Select(x => x.Item1)) { if (!changelistDict.TryGetValue(component, out var changelist)) continue; @@ -1143,7 +1136,7 @@ static async Task FullNotes(IGitHubClient client) return 0; } - static readonly HttpClient httpClient = new ( + static readonly HttpClient httpClient = new( new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate @@ -1192,7 +1185,8 @@ static async Task GenerateNotes(IGitHubClient client, Dictionary release.Id); var milestones = await TripleCheckGitHubPagination( - apiOptions => client.Issue.Milestone.GetAllForRepository(RepoOwner, RepoName, new MilestoneRequest { + apiOptions => client.Issue.Milestone.GetAllForRepository(RepoOwner, RepoName, new MilestoneRequest + { State = ItemStateFilter.All }, apiOptions), milestone => milestone.Id); @@ -1400,7 +1394,7 @@ static string GenerateComponentNotes(ReleaseNotes releaseNotes, Component compon PrintChanges(newNotes, relevantChangelog); } - if(component == Component.DreamMakerApi) + if (component == Component.DreamMakerApi) { newNotes.AppendLine(); newNotes.AppendLine("#tgs-dmapi-release"); @@ -1450,7 +1444,7 @@ static async Task ReleaseNuget(IGitHubClient client) { Component.NugetClient, "Client" }, }; - foreach(var kvp in csprojNameMap) + foreach (var kvp in csprojNameMap) { var component = kvp.Key; var csprojPath = CsprojSubstitution.Replace("$PROJECT$", kvp.Value); @@ -1588,7 +1582,7 @@ [optional blank line(s), stripped] builder.AppendLine(); builder.Append(" * The following changes are for "); builder.Append(GetComponentDisplayName(kvp.Key, true)); - if(kvp.Key == Component.Configuration) + if (kvp.Key == Component.Configuration) { builder.Append(". You "); if (kvp.Value.Version.Minor == 0 && kvp.Value.Version.Build == 0) @@ -1640,7 +1634,7 @@ [optional blank line(s), stripped] return 0; } - static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string pemBase64) + static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string pemBase64, bool toSS13) { var pemBytes = Convert.FromBase64String(pemBase64); var pem = Encoding.UTF8.GetString(pemBytes); @@ -1665,27 +1659,16 @@ static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string gitHubClient.Credentials = new Credentials(jwtStr, AuthenticationType.Bearer); - var installation = await gitHubClient.GitHubApps.GetRepositoryInstallationForCurrent(RepoOwner, RepoName); + var installation = await gitHubClient.GitHubApps.GetRepositoryInstallationForCurrent( + toSS13 + ? "spacestation13" + : RepoOwner, + RepoName); var installToken = await gitHubClient.GitHubApps.CreateInstallationToken(installation.Id); gitHubClient.Credentials = new Credentials(installToken.Token); } - static async ValueTask CICompletionCheck(GitHubClient gitHubClient, string currentSha, string pemBase64) - { - await GenerateAppCredentials(gitHubClient, pemBase64); - - await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Completion", currentSha) - { - CompletedAt = DateTime.UtcNow, - Conclusion = CheckConclusion.Success, - Output = new NewCheckRunOutput("CI Completion", "The CI Pipeline completed successfully"), - Status = CheckStatus.Completed, - }); - - return 0; - } - static void DebugAssert(bool condition, string message = null) { // This exists because one of the fucking asserts evaluates an enumerable or something and it was getting optimized out in release @@ -1695,5 +1678,21 @@ static void DebugAssert(bool condition, string message = null) else Debug.Assert(condition); } + + static async ValueTask CodecovCheck(IGitHubClient client, long runId) + { + var currentRun = await client.Actions.Workflows.Runs.Get(RepoOwner, RepoName, runId); + + bool foundRun = false; + for(int i = 0; i < 15 && !foundRun; ++i) + { + var allRuns = await client.Check.Run.GetAllForReference(RepoOwner, RepoName, currentRun.HeadSha); + foundRun = allRuns.CheckRuns.Any(x => x.CheckSuite.Id == currentRun.Id && x.Name == "codecov/project"); + if (!foundRun && i != 14) + await Task.Delay(TimeSpan.FromMinutes(1)); + } + + return foundRun ? 0 : 24398; + } } }