From 837881dcafeec20116d791b14c74745e600f7539 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 09:32:40 +0000 Subject: [PATCH 001/111] Bump Swashbuckle.AspNetCore.Newtonsoft from 6.6.2 to 6.7.0 Bumps [Swashbuckle.AspNetCore.Newtonsoft](https://github.com/domaindrivendev/Swashbuckle.AspNetCore) from 6.6.2 to 6.7.0. - [Release notes](https://github.com/domaindrivendev/Swashbuckle.AspNetCore/releases) - [Commits](https://github.com/domaindrivendev/Swashbuckle.AspNetCore/compare/v6.6.2...v6.7.0) --- updated-dependencies: - dependency-name: Swashbuckle.AspNetCore.Newtonsoft dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 197583214f0..06fd0d26100 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -117,7 +117,7 @@ - + From d903a8b42d8d25a8081c5912cf3a9198cf0463b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 18:58:43 +0000 Subject: [PATCH 002/111] Bump Swashbuckle.AspNetCore from 6.6.2 to 6.7.0 Bumps [Swashbuckle.AspNetCore](https://github.com/domaindrivendev/Swashbuckle.AspNetCore) from 6.6.2 to 6.7.0. - [Release notes](https://github.com/domaindrivendev/Swashbuckle.AspNetCore/releases) - [Commits](https://github.com/domaindrivendev/Swashbuckle.AspNetCore/compare/v6.6.2...v6.7.0) --- updated-dependencies: - dependency-name: Swashbuckle.AspNetCore dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 06fd0d26100..279c057c550 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -115,7 +115,7 @@ - + From dd048ef36eba6e0809ed8fbb43df29b0dadf8a27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 09:21:18 +0000 Subject: [PATCH 003/111] Bump MSTest.TestFramework from 3.5.0 to 3.5.1 Bumps [MSTest.TestFramework](https://github.com/microsoft/testfx) from 3.5.0 to 3.5.1. - [Release notes](https://github.com/microsoft/testfx/releases) - [Changelog](https://github.com/microsoft/testfx/blob/main/docs/Changelog.md) - [Commits](https://github.com/microsoft/testfx/compare/v3.5.0...v3.5.1) --- updated-dependencies: - dependency-name: MSTest.TestFramework dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build/TestCommon.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/TestCommon.props b/build/TestCommon.props index 1864c27e2a0..da1ad953f67 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -20,7 +20,7 @@ - + From ad4444761bd4a00abf8a24b73e81da9346617a8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 23:33:43 +0000 Subject: [PATCH 004/111] Bump MSTest.TestAdapter from 3.5.0 to 3.5.1 Bumps [MSTest.TestAdapter](https://github.com/microsoft/testfx) from 3.5.0 to 3.5.1. - [Release notes](https://github.com/microsoft/testfx/releases) - [Changelog](https://github.com/microsoft/testfx/blob/main/docs/Changelog.md) - [Commits](https://github.com/microsoft/testfx/compare/v3.5.0...v3.5.1) --- updated-dependencies: - dependency-name: MSTest.TestAdapter dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build/TestCommon.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/TestCommon.props b/build/TestCommon.props index da1ad953f67..98703bdbfd6 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -18,7 +18,7 @@ - + From 895212c265087acd596b223317ca12c264761e5a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 9 Aug 2024 18:33:08 -0400 Subject: [PATCH 005/111] DMAPI v7.2.0 - Add consumer APIs for overriding HTTP GET implementation. - Fix Discord documentation links. - Use IETF RFC 2119 wording in documentation. - Populate some documentation comments for previously undocumented public entities. --- build/Version.props | 2 +- src/DMAPI/tgs.dm | 100 +++++++++++++++++------ src/DMAPI/tgs/core/README.md | 2 +- src/DMAPI/tgs/core/byond_world_export.dm | 22 +++++ src/DMAPI/tgs/core/core.dm | 7 +- src/DMAPI/tgs/core/datum.dm | 2 +- src/DMAPI/tgs/includes.dm | 1 + src/DMAPI/tgs/v5/api.dm | 5 +- src/DMAPI/tgs/v5/bridge.dm | 21 ++--- 9 files changed, 117 insertions(+), 45 deletions(-) create mode 100644 src/DMAPI/tgs/core/byond_world_export.dm diff --git a/build/Version.props b/build/Version.props index 0099619aac5..c225e085ac1 100644 --- a/build/Version.props +++ b/build/Version.props @@ -9,7 +9,7 @@ 7.0.0 13.6.0 15.6.0 - 7.1.3 + 7.2.0 5.9.0 1.4.1 1.2.1 diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index 17464b44dae..fde85d1422e 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,18 +1,19 @@ // tgstation-server DMAPI +// The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in IETF RFC 2119. -#define TGS_DMAPI_VERSION "7.1.3" +#define TGS_DMAPI_VERSION "7.2.0" // All functions and datums outside this document are subject to change with any version and should not be relied on. // CONFIGURATION -/// Create this define if you want to do TGS configuration outside of this file. +/// Consumers SHOULD create this define if you want to do TGS configuration outside of this file. #ifndef TGS_EXTERNAL_CONFIGURATION -// Comment this out once you've filled in the below. +// Consumers MUST comment this out once you've filled in the below and are not using [TGS_EXTERNAL_CONFIGURATION]. #error TGS API unconfigured -// Uncomment this if you wish to allow the game to interact with TGS 3.. +// Consumers MUST uncomment this if you wish to allow the game to interact with TGS version 3. // This will raise the minimum required security level of your game to TGS_SECURITY_TRUSTED due to it utilizing call()(). //#define TGS_V3_API @@ -52,7 +53,7 @@ #ifndef TGS_FILE2TEXT_NATIVE #ifdef file2text -#error Your codebase is re-defining the BYOND proc file2text. The DMAPI requires the native version to read the result of world.Export(). You can fix this by adding "#define TGS_FILE2TEXT_NATIVE file2text" before your override of file2text to allow the DMAPI to use the native version. This will only be used for world.Export(), not regular file accesses +#error Your codebase is re-defining the BYOND proc file2text. The DMAPI requires the native version to read the result of world.Export(). You SHOULD fix this by adding "#define TGS_FILE2TEXT_NATIVE file2text" before your override of file2text to allow the DMAPI to use the native version. This will only be used for world.Export(), not regular file accesses #endif #define TGS_FILE2TEXT_NATIVE file2text #endif @@ -152,16 +153,17 @@ //REQUIRED HOOKS /** - * Call this somewhere in [/world/proc/New] that is always run. This function may sleep! + * Consumers MUST call this somewhere in [/world/proc/New] that is always run. This function may sleep! * * * event_handler - Optional user defined [/datum/tgs_event_handler]. * * minimum_required_security_level: The minimum required security level to run the game in which the DMAPI is integrated. Can be one of [TGS_SECURITY_ULTRASAFE], [TGS_SECURITY_SAFE], or [TGS_SECURITY_TRUSTED]. + * * http_handler - Optional user defined [/datum/tgs_http_handler]. */ -/world/proc/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE) +/world/proc/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE, datum/tgs_http_handler/http_handler) return /** - * Call this when your initializations are complete and your game is ready to play before any player interactions happen. + * Consumers MUST this when your initializations are complete and your game is ready to play before any player interactions happen. * * This may use [/world/var/sleep_offline] to make this happen so ensure no changes are made to it while this call is running. * Afterwards, consider explicitly setting it to what you want to avoid this BYOND bug: http://www.byond.com/forum/post/2575184 @@ -170,12 +172,10 @@ /world/proc/TgsInitializationComplete() return -/// Put this at the start of [/world/proc/Topic]. +/// Consumers MUST run this macro at the start of [/world/proc/Topic]. #define TGS_TOPIC var/tgs_topic_return = TgsTopic(args[1]); if(tgs_topic_return) return tgs_topic_return -/** - * Call this as late as possible in [world/proc/Reboot] (BEFORE ..()). - */ +/// Consumers MUST call this as late as possible in [world/proc/Reboot] (BEFORE ..()). /world/proc/TgsReboot() return @@ -269,7 +269,7 @@ /// The [/datum/tgs_chat_channel] the user was from. var/datum/tgs_chat_channel/channel -/// User definable handler for TGS events. +/// User definable handler for TGS events This abstract version SHOULD be overridden to be used. /datum/tgs_event_handler /// If the handler receieves [TGS_EVENT_HEALTH_CHECK] events. var/receive_health_checks = FALSE @@ -283,7 +283,41 @@ set waitfor = FALSE return -/// User definable chat command. +/// User definable handler for HTTP calls. This abstract version MUST be overridden to be used. +/datum/tgs_http_handler + +/** + * User definable callback for executing HTTP GET requests. + * MUST perform BYOND sleeps while the request is in flight. + * MUST return a [/datum/tgs_http_result]. + * SHOULD log its own errors + * + * url - The full URL to execute the GET request for including query parameters. + */ +/datum/tgs_http_handler/proc/PerformGet(url) + CRASH("[type]/PerformGet not implemented!") + +/// Result of a [/datum/tgs_http_handler] call. MUST NOT be overridden. +/datum/tgs_http_result + /// HTTP response as text + var/response_text + /// Boolean request success flag. Set for any 2XX response code. + var/success + +/** + * Create a [/datum/tgs_http_result]. + * + * * response_text - HTTP response as text. Must be provided in New(). + * * success - Boolean request success flag. Set for any 2XX response code. Must be provided in New(). + */ +/datum/tgs_http_result/New(response_text, success) + if(response_text && !istext(response_text)) + CRASH("response_text was not text!") + + src.response_text = response_text + src.success = success + +/// User definable chat command. This abstract version MUST be overridden to be used. /datum/tgs_chat_command /// The string to trigger this command on a chat bot. e.g `@bot name ...` or `!tgs name ...`. var/name = "" @@ -296,21 +330,27 @@ /** * Process command activation. Should return a [/datum/tgs_message_content] to respond to the issuer with. + * MUST be implemented * - * sender - The [/datum/tgs_chat_user] who issued the command. - * params - The trimmed string following the command `/datum/tgs_chat_command/var/name]. + * * sender - The [/datum/tgs_chat_user] who issued the command. + * * params - The trimmed string following the command `/datum/tgs_chat_command/var/name]. */ /datum/tgs_chat_command/proc/Run(datum/tgs_chat_user/sender, params) CRASH("[type] has no implementation for Run()") -/// User definable chat message. +/// User definable chat message. MUST NOT be overridden. /datum/tgs_message_content - /// The tring content of the message. Must be provided in New(). + /// The string content of the message. Must be provided in New(). var/text /// The [/datum/tgs_chat_embed] to embed in the message. Not supported on all chat providers. var/datum/tgs_chat_embed/structure/embed +/** + * Create a [/datum/tgs_message_content]. + * + * * text - The string content of the message. + */ /datum/tgs_message_content/New(text) ..() if(!istext(text)) @@ -319,7 +359,7 @@ src.text = text -/// User definable chat embed. Currently mirrors Discord chat embeds. See https://discord.com/developers/docs/resources/channel#embed-object-embed-structure for details. +/// User definable chat embed. Currently mirrors Discord chat embeds. See https://discord.com/developers/docs/resources/message#embed-object for details. /datum/tgs_chat_embed/structure var/title var/description @@ -331,13 +371,13 @@ /// Colour must be #AARRGGBB or #RRGGBB hex string. var/colour - /// See https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure for details. + /// See https://discord.com/developers/docs/resources/message#embed-object-embed-image-structure for details. var/datum/tgs_chat_embed/media/image - /// See https://discord.com/developers/docs/resources/channel#embed-object-embed-thumbnail-structure for details. + /// See https://discord.com/developers/docs/resources/message#embed-object-embed-thumbnail-structure for details. var/datum/tgs_chat_embed/media/thumbnail - /// See https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure for details. + /// See https://discord.com/developers/docs/resources/message#embed-object-embed-video-structure for details. var/datum/tgs_chat_embed/media/video var/datum/tgs_chat_embed/footer/footer @@ -346,7 +386,7 @@ var/list/datum/tgs_chat_embed/field/fields -/// Common datum for similar discord embed medias. +/// Common datum for similar Discord embed medias. /datum/tgs_chat_embed/media /// Must be set in New(). var/url @@ -354,6 +394,7 @@ var/height var/proxy_url +/// Create a [/datum/tgs_chat_embed]. /datum/tgs_chat_embed/media/New(url) ..() if(!istext(url)) @@ -361,13 +402,14 @@ src.url = url -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-footer-structure for details. +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-footer-structure for details. /datum/tgs_chat_embed/footer /// Must be set in New(). var/text var/icon_url var/proxy_icon_url +/// Create a [/datum/tgs_chat_embed/footer]. /datum/tgs_chat_embed/footer/New(text) ..() if(!istext(text)) @@ -375,16 +417,17 @@ src.text = text -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-provider-structure for details. +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-provider-structure for details. /datum/tgs_chat_embed/provider var/name var/url -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-author-structure for details. Must have name set in New(). +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-author-structure for details. Must have name set in New(). /datum/tgs_chat_embed/provider/author var/icon_url var/proxy_icon_url +/// Create a [/datum/tgs_chat_embed/footer]. /datum/tgs_chat_embed/provider/author/New(name) ..() if(!istext(name)) @@ -392,12 +435,15 @@ src.name = name -/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-field-structure for details. Must have name and value set in New(). +/// See https://discord.com/developers/docs/resources/message#embed-object-embed-field-structure for details. /datum/tgs_chat_embed/field + /// Must be set in New(). var/name + /// Must be set in New(). var/value var/is_inline +/// Create a [/datum/tgs_chat_embed/field]. /datum/tgs_chat_embed/field/New(name, value) ..() if(!istext(name)) diff --git a/src/DMAPI/tgs/core/README.md b/src/DMAPI/tgs/core/README.md index b82d8f49e29..965e21b549a 100644 --- a/src/DMAPI/tgs/core/README.md +++ b/src/DMAPI/tgs/core/README.md @@ -3,7 +3,7 @@ This folder contains all DMAPI code not directly involved in an API. - [_definitions.dm](./definitions.dm) contains defines needed across DMAPI internals. +- [byond_world_export.dm](./byond_world_export.dm) contains the default `/datum/tgs_http_handler` implementation which uses `world.Export()`. - [core.dm](./core.dm) contains the implementations of the `/world/proc/TgsXXX()` procs. Many map directly to the `/datum/tgs_api` functions. It also contains the /datum selection and setup code. - [datum.dm](./datum.dm) contains the `/datum/tgs_api` declarations that all APIs must implement. - [tgs_version.dm](./tgs_version.dm) contains the `/datum/tgs_version` definition -- diff --git a/src/DMAPI/tgs/core/byond_world_export.dm b/src/DMAPI/tgs/core/byond_world_export.dm new file mode 100644 index 00000000000..6ef8d841b8f --- /dev/null +++ b/src/DMAPI/tgs/core/byond_world_export.dm @@ -0,0 +1,22 @@ +/datum/tgs_http_handler/byond_world_export + +/datum/tgs_http_handler/byond_world_export/PerformGet(url) + // This is an infinite sleep until we get a response + var/export_response = world.Export(url) + TGS_DEBUG_LOG("byond_world_export: Export complete") + + if(!export_response) + TGS_ERROR_LOG("byond_world_export: Failed request: [url]") + return new /datum/tgs_http_result(null, FALSE) + + var/content = export_response["CONTENT"] + if(!content) + TGS_ERROR_LOG("byond_world_export: Failed request, missing content!") + return new /datum/tgs_http_result(null, FALSE) + + var/response_json = TGS_FILE2TEXT_NATIVE(content) + if(!response_json) + TGS_ERROR_LOG("byond_world_export: Failed request, failed to load content!") + return new /datum/tgs_http_result(null, FALSE) + + return new /datum/tgs_http_result(response_json, TRUE) diff --git a/src/DMAPI/tgs/core/core.dm b/src/DMAPI/tgs/core/core.dm index 15622228e91..63cb5a2c351 100644 --- a/src/DMAPI/tgs/core/core.dm +++ b/src/DMAPI/tgs/core/core.dm @@ -1,4 +1,4 @@ -/world/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE) +/world/TgsNew(datum/tgs_event_handler/event_handler, minimum_required_security_level = TGS_SECURITY_ULTRASAFE, datum/tgs_http_handler/http_handler = null) var/current_api = TGS_READ_GLOBAL(tgs) if(current_api) TGS_ERROR_LOG("API datum already set (\ref[current_api] ([current_api]))! Was TgsNew() called more than once?") @@ -55,7 +55,10 @@ TGS_ERROR_LOG("Invalid parameter for event_handler: [event_handler]") event_handler = null - var/datum/tgs_api/new_api = new api_datum(event_handler, version) + if(!http_handler) + http_handler = new /datum/tgs_http_handler/byond_world_export + + var/datum/tgs_api/new_api = new api_datum(event_handler, version, http_handler) TGS_WRITE_GLOBAL(tgs, new_api) diff --git a/src/DMAPI/tgs/core/datum.dm b/src/DMAPI/tgs/core/datum.dm index f734fd0527f..3ca53e9bf7c 100644 --- a/src/DMAPI/tgs/core/datum.dm +++ b/src/DMAPI/tgs/core/datum.dm @@ -6,7 +6,7 @@ TGS_DEFINE_AND_SET_GLOBAL(tgs, null) var/list/warned_deprecated_command_runs -/datum/tgs_api/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version) +/datum/tgs_api/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version, datum/tgs_http_handler/http_handler) ..() src.event_handler = event_handler src.version = version diff --git a/src/DMAPI/tgs/includes.dm b/src/DMAPI/tgs/includes.dm index 23b714f9d06..f5118ed55a3 100644 --- a/src/DMAPI/tgs/includes.dm +++ b/src/DMAPI/tgs/includes.dm @@ -1,4 +1,5 @@ #include "core\_definitions.dm" +#include "core\byond_world_export.dm" #include "core\core.dm" #include "core\datum.dm" #include "core\tgs_version.dm" diff --git a/src/DMAPI/tgs/v5/api.dm b/src/DMAPI/tgs/v5/api.dm index 05d0dee25b3..3e328fc7c27 100644 --- a/src/DMAPI/tgs/v5/api.dm +++ b/src/DMAPI/tgs/v5/api.dm @@ -31,9 +31,12 @@ var/detached = FALSE -/datum/tgs_api/v5/New() + var/datum/tgs_http_handler/http_handler + +/datum/tgs_api/v5/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version, datum/tgs_http_handler/http_handler) . = ..() interop_version = version + src.http_handler = http_handler TGS_DEBUG_LOG("V5 API created: [json_encode(args)]") /datum/tgs_api/v5/ApiVersion() diff --git a/src/DMAPI/tgs/v5/bridge.dm b/src/DMAPI/tgs/v5/bridge.dm index 0c5e701a32b..62201fcc9e5 100644 --- a/src/DMAPI/tgs/v5/bridge.dm +++ b/src/DMAPI/tgs/v5/bridge.dm @@ -78,27 +78,24 @@ WaitForReattach(FALSE) TGS_DEBUG_LOG("Bridge request start") - // This is an infinite sleep until we get a response - var/export_response = world.Export(bridge_request) + var/datum/tgs_http_result/result = http_handler.PerformGet(bridge_request) TGS_DEBUG_LOG("Bridge request complete") - if(!export_response) - TGS_ERROR_LOG("Failed bridge request: [bridge_request]") + if(isnull(result)) + TGS_ERROR_LOG("Failed bridge request, handler returned null!") return - var/content = export_response["CONTENT"] - if(!content) - TGS_ERROR_LOG("Failed bridge request, missing content!") + if(!istype(result) || result.type != /datum/tgs_http_result) + TGS_ERROR_LOG("Failed bridge request, handler returned non-[/datum/tgs_http_result]!") return - var/response_json = TGS_FILE2TEXT_NATIVE(content) - if(!response_json) - TGS_ERROR_LOG("Failed bridge request, failed to load content!") + if(!result.success) + TGS_DEBUG_LOG("Failed bridge request, HTTP request failed!") return - var/list/bridge_response = json_decode(response_json) + var/list/bridge_response = json_decode(result.response_text) if(!bridge_response) - TGS_ERROR_LOG("Failed bridge request, bad json: [response_json]") + TGS_ERROR_LOG("Failed bridge request, bad json: [result.response_text]") return var/error = bridge_response[DMAPI5_RESPONSE_ERROR_MESSAGE] From 8aa1522b7d0c141d4fdf5fd4276c0d19b6178319 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 9 Aug 2024 22:26:44 -0400 Subject: [PATCH 006/111] Bump to MariaDB 11.4.2 --- build/Version.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index c225e085ac1..74c77f27a71 100644 --- a/build/Version.props +++ b/build/Version.props @@ -18,9 +18,9 @@ 8 https://download.visualstudio.microsoft.com/download/pr/751d3fcd-72db-4da2-b8d0-709c19442225/33cc492bde704bfd6d70a2b9109005a0/dotnet-hosting-8.0.6-win.exe - 10.11.8 + 11.4.2 - https://mirror.its.dal.ca/mariadb//mariadb-10.11.8/winx64-packages/mariadb-10.11.8-winx64.msi + 1.22.21 From bd1deb0a9b9fdd45cb10d39311fa2783be60440c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 10 Aug 2024 17:18:45 -0400 Subject: [PATCH 007/111] Add 30 minute cache to GitHub releases requests Closes #1831 --- build/Version.props | 6 +- .../Models/Response/AdministrationResponse.cs | 5 ++ .../AdministrationClient.cs | 2 +- .../IAdministrationClient.cs | 3 +- .../Controllers/AdministrationController.cs | 76 +++++++++++++------ .../Live/AdministrationTest.cs | 22 +++++- .../Live/Instance/JobsHubTests.cs | 2 +- .../Live/RawRequestTests.cs | 2 +- .../Live/TestLiveServer.cs | 2 +- 9 files changed, 86 insertions(+), 34 deletions(-) diff --git a/build/Version.props b/build/Version.props index 74c77f27a71..6e455640947 100644 --- a/build/Version.props +++ b/build/Version.props @@ -5,10 +5,10 @@ 6.8.0 5.1.0 - 10.6.0 + 10.7.0 7.0.0 - 13.6.0 - 15.6.0 + 13.7.0 + 16.0.0 7.2.0 5.9.0 1.4.1 diff --git a/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs b/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs index 55d7f646e5c..2db5d354a75 100644 --- a/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs +++ b/src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs @@ -16,5 +16,10 @@ public sealed class AdministrationResponse /// The latest available version of the Tgstation.Server.Host assembly from the upstream repository. If is not equal to 4 the update cannot be applied due to API changes. /// public Version? LatestVersion { get; set; } + + /// + /// This response is cached. This field indicates the when it was generated. + /// + public DateTimeOffset? GeneratedAt { get; set; } } } diff --git a/src/Tgstation.Server.Client/AdministrationClient.cs b/src/Tgstation.Server.Client/AdministrationClient.cs index 2d6841795cd..68c7fecd345 100644 --- a/src/Tgstation.Server.Client/AdministrationClient.cs +++ b/src/Tgstation.Server.Client/AdministrationClient.cs @@ -24,7 +24,7 @@ public AdministrationClient(IApiClient apiClient) } /// - public ValueTask Read(CancellationToken cancellationToken) => ApiClient.Read(Routes.Administration, cancellationToken); + public ValueTask Read(bool forceFresh, CancellationToken cancellationToken) => ApiClient.Read($"{Routes.Administration}?fresh={forceFresh}", cancellationToken); /// public async ValueTask Update( diff --git a/src/Tgstation.Server.Client/IAdministrationClient.cs b/src/Tgstation.Server.Client/IAdministrationClient.cs index 38d08983450..6d5d88f9510 100644 --- a/src/Tgstation.Server.Client/IAdministrationClient.cs +++ b/src/Tgstation.Server.Client/IAdministrationClient.cs @@ -17,9 +17,10 @@ public interface IAdministrationClient /// /// Get the represented by the . /// + /// If the response will be forcefully regenerated. /// The for the operation. /// A resulting in the represented by the . - ValueTask Read(CancellationToken cancellationToken); + ValueTask Read(bool forceFresh = false, CancellationToken cancellationToken = default); /// /// Updates the setttings. diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index 2ccc4f4c8bc..d4c1335df77 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -7,6 +7,7 @@ using System.Web; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -42,6 +43,11 @@ public sealed class AdministrationController : ApiController /// const string OctokitException = "Bad GitHub API response, check configuration!"; + /// + /// The key for . + /// + static readonly object ReadCacheKey = new(); + /// /// The for the . /// @@ -77,6 +83,11 @@ public sealed class AdministrationController : ApiController /// readonly IFileTransferTicketProvider fileTransferService; + /// + /// The for the . + /// + readonly IMemoryCache cacheService; + /// /// The for the . /// @@ -94,6 +105,7 @@ public sealed class AdministrationController : ApiController /// The value of . /// The value of . /// The value of . + /// The value of . /// The for the . /// The containing value of . /// The for the . @@ -107,6 +119,7 @@ public AdministrationController( IIOManager ioManager, IPlatformIdentifier platformIdentifier, IFileTransferTicketProvider fileTransferService, + IMemoryCache cacheService, ILogger logger, IOptions fileLoggingConfigurationOptions, IApiHeadersProvider apiHeadersProvider) @@ -124,12 +137,14 @@ public AdministrationController( this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); + this.cacheService = cacheService ?? throw new ArgumentNullException(nameof(cacheService)); fileLoggingConfiguration = fileLoggingConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(fileLoggingConfigurationOptions)); } /// /// Get server information. /// + /// If , the cache should be bypassed. /// The for the operation. /// A resulting in the for the operation. /// Retrieved data successfully. @@ -140,39 +155,56 @@ public AdministrationController( [ProducesResponseType(typeof(AdministrationResponse), 200)] [ProducesResponseType(typeof(ErrorMessageResponse), 424)] [ProducesResponseType(typeof(ErrorMessageResponse), 429)] - public async ValueTask Read(CancellationToken cancellationToken) + public async ValueTask Read([FromQuery] bool? fresh, CancellationToken cancellationToken) { try { - Version? greatestVersion = null; - Uri? repoUrl = null; - try + async Task CacheFactory() { - var gitHubService = gitHubServiceFactory.CreateService(); - var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken); - var releases = await gitHubService.GetTgsReleases(cancellationToken); - - foreach (var kvp in releases) + Version? greatestVersion = null; + Uri? repoUrl = null; + try + { + var gitHubService = gitHubServiceFactory.CreateService(); + var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken); + var releases = await gitHubService.GetTgsReleases(cancellationToken); + + foreach (var kvp in releases) + { + var version = kvp.Key; + var release = kvp.Value; + if (version.Major > 3 // Forward/backward compatible but not before TGS4 + && (greatestVersion == null || version > greatestVersion)) + greatestVersion = version; + } + + repoUrl = await repositoryUrlTask; + } + catch (NotFoundException e) { - var version = kvp.Key; - var release = kvp.Value; - if (version.Major > 3 // Forward/backward compatible but not before TGS4 - && (greatestVersion == null || version > greatestVersion)) - greatestVersion = version; + Logger.LogWarning(e, "Not found exception while retrieving upstream repository info!"); } - repoUrl = await repositoryUrlTask; + return Json(new AdministrationResponse + { + LatestVersion = greatestVersion, + TrackedRepositoryUrl = repoUrl, + GeneratedAt = DateTimeOffset.UtcNow, + }); } - catch (NotFoundException e) + + var ttl = TimeSpan.FromMinutes(30); + Task task; + if (fresh == true || !cacheService.TryGetValue(ReadCacheKey, out var rawCacheObject)) { - Logger.LogWarning(e, "Not found exception while retrieving upstream repository info!"); + using var entry = cacheService.CreateEntry(ReadCacheKey); + entry.AbsoluteExpirationRelativeToNow = ttl; + entry.Value = task = CacheFactory(); } + else + task = (Task)rawCacheObject!; - return Json(new AdministrationResponse - { - LatestVersion = greatestVersion, - TrackedRepositoryUrl = repoUrl, - }); + return await task; } catch (RateLimitExceededException e) { diff --git a/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs b/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs index adfb009d697..afeabef6f5d 100644 --- a/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/AdministrationTest.cs @@ -1,10 +1,10 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; +using System; using System.IO; -using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Client; @@ -54,10 +54,24 @@ await ApiAssert.ThrowsException(() => badClient.Administration.Read(cancellationToken)); + await ApiAssert.ThrowsException(() => badClient.Administration.Read(false, cancellationToken)); await ApiAssert.ThrowsException(() => badClient.ServerInformation(cancellationToken)); } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index bfc4538c685..b34b14f066d 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -670,7 +670,7 @@ await Task.WhenAny( "asdfasdfasdfasdf"); await using var node1BadClient = clientFactory.CreateFromToken(node1.RootUrl, controllerUserClient.Token); - await ApiAssert.ThrowsException(() => node1BadClient.Administration.Read(cancellationToken)); + await ApiAssert.ThrowsException(() => node1BadClient.Administration.Read(false, cancellationToken)); // check instance info is not shared var controllerInstance = await controllerClient.Instances.CreateOrAttach( From f7c94618b2bd09ff679edcb10d218e19bdbd2358 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 10 Aug 2024 22:54:04 -0400 Subject: [PATCH 008/111] Fix DMAPI documentation comment typos --- build/Version.props | 2 +- src/DMAPI/tgs.dm | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/Version.props b/build/Version.props index 74c77f27a71..6b80b4ff792 100644 --- a/build/Version.props +++ b/build/Version.props @@ -9,7 +9,7 @@ 7.0.0 13.6.0 15.6.0 - 7.2.0 + 7.2.1 5.9.0 1.4.1 1.2.1 diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index fde85d1422e..4766b3dfe66 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,7 +1,7 @@ // tgstation-server DMAPI // The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in IETF RFC 2119. -#define TGS_DMAPI_VERSION "7.2.0" +#define TGS_DMAPI_VERSION "7.2.1" // All functions and datums outside this document are subject to change with any version and should not be relied on. @@ -163,7 +163,7 @@ return /** - * Consumers MUST this when your initializations are complete and your game is ready to play before any player interactions happen. + * Consumers MUST call this when world initializations are complete and the game is ready to play before any player interactions happen. * * This may use [/world/var/sleep_offline] to make this happen so ensure no changes are made to it while this call is running. * Afterwards, consider explicitly setting it to what you want to avoid this BYOND bug: http://www.byond.com/forum/post/2575184 From 792947da4dca3b09b38af323fe2a754b383546c6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 10 Aug 2024 17:23:07 -0400 Subject: [PATCH 009/111] Add configuration option to force use of dreamdaemon.exe --- build/Version.props | 2 +- .../Components/Engine/WindowsByondInstaller.cs | 2 +- .../Configuration/SessionConfiguration.cs | 5 +++++ src/Tgstation.Server.Host/appsettings.yml | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index 6e455640947..c947fc3c5c2 100644 --- a/build/Version.props +++ b/build/Version.props @@ -4,7 +4,7 @@ 6.8.0 - 5.1.0 + 5.2.0 10.7.0 7.0.0 13.7.0 diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs index 35bcbf63dc0..ce2c7704e22 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs @@ -224,7 +224,7 @@ public override async ValueTask TrustDmbPath(EngineVersion version, string fullD /// protected override string GetDreamDaemonName(Version byondVersion, out bool supportsCli) { - supportsCli = byondVersion >= DDExeVersion; + supportsCli = byondVersion >= DDExeVersion && !sessionConfiguration.ForceUseDreamDaemonExe; return supportsCli ? "dd.exe" : "dreamdaemon.exe"; } diff --git a/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs b/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs index c543698b905..d7ee3e1c9d2 100644 --- a/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs @@ -29,5 +29,10 @@ sealed class SessionConfiguration /// If , deployments that fail will not be immediately cleaned up. They will be cleaned up the next time the instance is onlined. /// public bool DelayCleaningFailedDeployments { get; set; } + + /// + /// If set dd.exe will not be used on Windows systems in versions where it is present. Instead dreamdaemon.exe will always be used. + /// + public bool ForceUseDreamDaemonExe { get; set; } } } diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 4868f519fd3..526f38ab386 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -23,6 +23,7 @@ Session: HighPriorityLiveDreamDaemon: false # If DreamDaemon instances should run as higher priority processes LowPriorityDeploymentProcesses: true # If TGS Deployments should run as lower priority processes DelayCleaningFailedDeployments: false # If true, deployments that fail will not be immediately cleaned up. They will be cleaned up the next time the instance is onlined + ForceUseDreamDaemonExe: false # If true, dd.exe will not be used on Windows systems in versions where it is present. Instead dreamdaemon.exe will always be used. FileLogging: Directory: # Directory in which log files are stored. Windows default: %PROGRAMDATA%/tgstation-server. Linux default: /var/log/tgstation-server Disable: true # Disable file logging entirely From 788fbb592101c9e12ed71247a6afd167d68b0695 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 10 Aug 2024 19:05:32 -0400 Subject: [PATCH 010/111] Minor code cleanups --- .../Chat/Commands/RevisionCommand.cs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs b/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs index 39ed128d591..1bd2a8b91e0 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs @@ -50,7 +50,7 @@ public RevisionCommand(IWatchdog watchdog, IRepositoryManager repositoryManager) public async ValueTask Invoke(string arguments, ChatUser user, CancellationToken cancellationToken) { string result; - if (arguments.Split(' ').Any(x => x.ToUpperInvariant() == "--REPO")) + if (arguments.Split(' ').Any(x => x.Equals("--repo", StringComparison.OrdinalIgnoreCase))) { if (repositoryManager.CloneInProgress || repositoryManager.InUse) return new MessageContent @@ -58,15 +58,13 @@ public async ValueTask Invoke(string arguments, ChatUser user, C Text = "Repository busy! Try again later", }; - using (var repo = await repositoryManager.LoadRepository(cancellationToken)) - { - if (repo == null) - return new MessageContent - { - Text = "Repository unavailable!", - }; - result = repo.Head; - } + using var repo = await repositoryManager.LoadRepository(cancellationToken); + if (repo == null) + return new MessageContent + { + Text = "Repository unavailable!", + }; + result = repo.Head; } else { From 89d70be78207632267e7dd5d1135f84c26dd1784 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 10 Aug 2024 19:06:10 -0400 Subject: [PATCH 011/111] Implement repository recloning Closes #1016 --- src/Tgstation.Server.Api/Models/JobCode.cs | 6 ++ .../Rights/RepositoryRights.cs | 5 + .../Components/IRepositoryClient.cs | 7 ++ .../Components/RepositoryClient.cs | 3 + .../Components/Engine/OpenDreamInstaller.cs | 1 + .../Components/Events/EventType.cs | 2 +- .../Components/Repository/IRepository.cs | 2 + .../Components/Repository/Repository.cs | 67 ++++++++---- .../Repository/RepositoryUpdateService.cs | 102 +++++++++++++++--- .../Controllers/RepositoryController.cs | 57 ++++++++-- .../Live/Instance/RepositoryTest.cs | 19 ++++ .../Tgstation.Server.Tests/TestRepository.cs | 2 +- 12 files changed, 228 insertions(+), 45 deletions(-) diff --git a/src/Tgstation.Server.Api/Models/JobCode.cs b/src/Tgstation.Server.Api/Models/JobCode.cs index be5db8d5a6b..1c20107b59f 100644 --- a/src/Tgstation.Server.Api/Models/JobCode.cs +++ b/src/Tgstation.Server.Api/Models/JobCode.cs @@ -108,5 +108,11 @@ public enum JobCode : byte /// [Description("Reconnect chat bot")] ReconnectChatBot, + + /// + /// When a repository is recloned. + /// + [Description("Reclone repository")] + RepositoryReclone, } } diff --git a/src/Tgstation.Server.Api/Rights/RepositoryRights.cs b/src/Tgstation.Server.Api/Rights/RepositoryRights.cs index f9d134820bf..407437ce5d1 100644 --- a/src/Tgstation.Server.Api/Rights/RepositoryRights.cs +++ b/src/Tgstation.Server.Api/Rights/RepositoryRights.cs @@ -82,5 +82,10 @@ public enum RepositoryRights : ulong /// User may change submodule update settings. /// ChangeSubmoduleUpdate = 1 << 13, + + /// + /// User may trigger repository recloning. + /// + Reclone = 1 << 14, } } diff --git a/src/Tgstation.Server.Client/Components/IRepositoryClient.cs b/src/Tgstation.Server.Client/Components/IRepositoryClient.cs index 9d3bef806cf..51d8bdfd805 100644 --- a/src/Tgstation.Server.Client/Components/IRepositoryClient.cs +++ b/src/Tgstation.Server.Client/Components/IRepositoryClient.cs @@ -40,5 +40,12 @@ public interface IRepositoryClient /// The for the operation. /// A resulting in the . ValueTask Delete(CancellationToken cancellationToken); + + /// + /// Deletes and reclones the repository. + /// + /// The for the operation. + /// A resulting in the . + ValueTask Reclone(CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Client/Components/RepositoryClient.cs b/src/Tgstation.Server.Client/Components/RepositoryClient.cs index 124f1ec5b28..57116120247 100644 --- a/src/Tgstation.Server.Client/Components/RepositoryClient.cs +++ b/src/Tgstation.Server.Client/Components/RepositoryClient.cs @@ -44,5 +44,8 @@ public RepositoryClient(IApiClient apiClient, Instance instance) /// public ValueTask Update(RepositoryUpdateRequest repository, CancellationToken cancellationToken) => apiClient.Update(Routes.Repository, repository ?? throw new ArgumentNullException(nameof(repository)), instance.Id!.Value, cancellationToken); + + /// + public ValueTask Reclone(CancellationToken cancellationToken) => apiClient.Patch(Routes.Repository, instance.Id!.Value, cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index e552d16b406..f99d70bfa48 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -176,6 +176,7 @@ await repo.CheckoutObject( null, null, true, + false, progressSection2, cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index 5d8c0b9054d..711d6ca246d 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -12,7 +12,7 @@ public enum EventType RepoResetOrigin, /// - /// Parameters: Checkout target. + /// Parameters: Checkout target, hard reset flag (If "True", this is actually a hard reset, not a checkout). /// [EventScript("RepoCheckout")] RepoCheckout, diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs index 65bcd1d61ed..e736c4873e1 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs @@ -47,6 +47,7 @@ public interface IRepository : IGitRemoteAdditionalInformation, IDisposable /// The optional username used for fetching from submodule repositories. /// The optional password used for fetching from submodule repositories. /// If a submodule update should be attempted after the merge. + /// If a hard reset to the target committish should be performed instead of a checkout. /// The optional to report progress of the operation. /// The for the operation. /// A representing the running operation. @@ -55,6 +56,7 @@ ValueTask CheckoutObject( string? username, string? password, bool updateSubmodules, + bool moveCurrentReference, JobProgressReporter? progressReporter, CancellationToken cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 972fae31cca..6873d429578 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -295,7 +295,7 @@ await Task.Factory.StartNew( var revertTo = originalCommit.CanonicalName ?? originalCommit.Tip.Sha; logger.LogDebug("Merge conflict, aborting and reverting to {revertTarget}", revertTo); progressReporter.ReportProgress(0); - RawCheckout(revertTo, progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0), cancellationToken); + RawCheckout(revertTo, false, progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0), cancellationToken); cancellationToken.ThrowIfCancellationRequested(); } @@ -376,19 +376,21 @@ public async ValueTask CheckoutObject( string? username, string? password, bool updateSubmodules, + bool moveCurrentReference, JobProgressReporter? progressReporter, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(committish); logger.LogDebug("Checkout object: {committish}...", committish); - await eventConsumer.HandleEvent(EventType.RepoCheckout, new List { committish }, false, cancellationToken); + await eventConsumer.HandleEvent(EventType.RepoCheckout, new List { committish, moveCurrentReference.ToString() }, false, cancellationToken); await Task.Factory.StartNew( () => { libGitRepo.RemoveUntrackedFiles(); RawCheckout( committish, + moveCurrentReference, progressReporter?.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0), cancellationToken); }, @@ -879,9 +881,10 @@ protected override void DisposeImpl() /// Runs a blocking force checkout to . /// /// The committish to checkout. + /// If a hard reset should actually be performed. /// The optional for the operation. /// The for the operation. - void RawCheckout(string committish, JobProgressReporter? progressReporter, CancellationToken cancellationToken) + void RawCheckout(string committish, bool moveCurrentReference, JobProgressReporter? progressReporter, CancellationToken cancellationToken) { logger.LogTrace("Checkout: {committish}", committish); @@ -900,34 +903,52 @@ void RawCheckout(string committish, JobProgressReporter? progressReporter, Cance cancellationToken.ThrowIfCancellationRequested(); - void RunCheckout() => commands.Checkout( - libGitRepo, - checkoutOptions, - committish); - - try + if (moveCurrentReference) { - RunCheckout(); + if (Reference == NoReference) + throw new InvalidOperationException("Cannot move current reference when not on reference!"); + + var gitObject = libGitRepo.Lookup(committish); + if (gitObject == null) + throw new JobException($"Could not find committish: {committish}"); + + var commit = gitObject.Peel(); + + cancellationToken.ThrowIfCancellationRequested(); + + libGitRepo.Reset(ResetMode.Hard, commit, checkoutOptions); } - catch (NotFoundException) + else { - // Maybe (likely) a remote? - var remoteName = $"origin/{committish}"; - var remoteBranch = libGitRepo.Branches.FirstOrDefault( - branch => branch.FriendlyName.Equals(remoteName, StringComparison.Ordinal)); - cancellationToken.ThrowIfCancellationRequested(); + void RunCheckout() => commands.Checkout( + libGitRepo, + checkoutOptions, + committish); + + try + { + RunCheckout(); + } + catch (NotFoundException) + { + // Maybe (likely) a remote? + var remoteName = $"origin/{committish}"; + var remoteBranch = libGitRepo.Branches.FirstOrDefault( + branch => branch.FriendlyName.Equals(remoteName, StringComparison.Ordinal)); + cancellationToken.ThrowIfCancellationRequested(); - if (remoteBranch == default) - throw; + if (remoteBranch == default) + throw; - logger.LogDebug("Creating local branch for {remoteBranchFriendlyName}...", remoteBranch.FriendlyName); - var branch = libGitRepo.CreateBranch(committish, remoteBranch.Tip); + logger.LogDebug("Creating local branch for {remoteBranchFriendlyName}...", remoteBranch.FriendlyName); + var branch = libGitRepo.CreateBranch(committish, remoteBranch.Tip); - libGitRepo.Branches.Update(branch, branchUpdate => branchUpdate.TrackedBranch = remoteBranch.CanonicalName); + libGitRepo.Branches.Update(branch, branchUpdate => branchUpdate.TrackedBranch = remoteBranch.CanonicalName); - cancellationToken.ThrowIfCancellationRequested(); + cancellationToken.ThrowIfCancellationRequested(); - RunCheckout(); + RunCheckout(); + } } cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs index a7aca78c4b8..9861a854183 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs @@ -22,11 +22,6 @@ namespace Tgstation.Server.Host.Components.Repository /// sealed class RepositoryUpdateService { - /// - /// The for the . - /// - readonly RepositoryUpdateRequest model; - /// /// The current for the . /// @@ -50,19 +45,16 @@ sealed class RepositoryUpdateService /// /// Initializes a new instance of the class. /// - /// The value of . /// The value of . /// The value of . /// The value of . /// The value of . public RepositoryUpdateService( - RepositoryUpdateRequest model, RepositorySettings currentModel, User initiatingUser, ILogger logger, long instanceId) { - this.model = model ?? throw new ArgumentNullException(nameof(model)); this.currentModel = currentModel ?? throw new ArgumentNullException(nameof(currentModel)); this.initiatingUser = initiatingUser ?? throw new ArgumentNullException(nameof(initiatingUser)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -139,24 +131,25 @@ public static async ValueTask LoadRevisionInformation( /// /// The job entrypoint used by to update the repository's current HEAD. /// - /// The the job is running on. only when performing an instance move operation. + /// The . + /// The the job is running on. /// The for the operation. - /// The running , ignored. /// The for the job. /// The for the operation. /// A representing the running operation. #pragma warning disable CA1502, CA1506 // TODO: Decomplexify public async ValueTask RepositoryUpdateJob( + RepositoryUpdateRequest model, IInstanceCore? instance, IDatabaseContextFactory databaseContextFactory, - Job job, JobProgressReporter progressReporter, CancellationToken cancellationToken) #pragma warning restore CA1502, CA1506 { + ArgumentNullException.ThrowIfNull(model); ArgumentNullException.ThrowIfNull(instance); - - _ = job; // shuts up an IDE warning + ArgumentNullException.ThrowIfNull(databaseContextFactory); + ArgumentNullException.ThrowIfNull(progressReporter); var repoManager = instance.RepositoryManager; using var repo = await repoManager.LoadRepository(cancellationToken) ?? throw new JobException(ErrorCode.RepoMissing); @@ -315,6 +308,7 @@ await repo.CheckoutObject( currentModel.AccessUser, currentModel.AccessToken, updateSubmodules, + false, NextProgressReporter("Checkout"), cancellationToken); await CallLoadRevInfo(); // we've either seen origin before or what we're checking out is on origin @@ -579,6 +573,7 @@ await repo.CheckoutObject( currentModel.AccessUser, currentModel.AccessToken, true, + false, progressReporter.CreateSection($"Checkout {startReference ?? startSha[..7]}", secondStep ? 0.5 : 1.0), default); @@ -588,5 +583,86 @@ await repo.CheckoutObject( throw; } } + + /// + /// The job entrypoint used by to reclone a repository. + /// + /// The the job is running on. + /// The for the operation. + /// The for the job. + /// The for the operation. + /// A representing the running operation. + public async ValueTask RepositoryRecloneJob( + IInstanceCore? instance, + IDatabaseContextFactory databaseContextFactory, + JobProgressReporter progressReporter, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(instance); + ArgumentNullException.ThrowIfNull(databaseContextFactory); + ArgumentNullException.ThrowIfNull(progressReporter); + + progressReporter.StageName = "Loading Old Repository"; + + Uri origin; + string? oldReference; + string oldSha; + ValueTask deleteTask; + using (var oldRepo = await instance.RepositoryManager.LoadRepository(cancellationToken)) + { + if (oldRepo == null) + throw new JobException(ErrorCode.RepoMissing); + + origin = oldRepo.Origin; + oldSha = oldRepo.Head; + oldReference = oldRepo.Reference; + if (oldReference == Repository.NoReference) + oldReference = null; + + progressReporter.StageName = "Deleting Old Repository"; + deleteTask = instance.RepositoryManager.DeleteRepository(cancellationToken); + } + + await deleteTask; + progressReporter.ReportProgress(0.1); + IRepository newRepo; + try + { + newRepo = await instance.RepositoryManager.CloneRepository( + origin, + oldReference, + currentModel.AccessUser, + currentModel.AccessToken, + progressReporter.CreateSection("Cloning New Repository", 0.8), + true, // TODO: Make configurable maybe... + cancellationToken) + ?? throw new JobException("A race condition occurred while recloning the repository. Somehow, it was fully cloned instantly after being deleted!"); // I'll take lines of code that should never be hit for $10k + } + catch (Exception ex) when (ex is not JobException) + { + logger.LogWarning("Reclone failed, clearing credentials!"); + + // need to clear credentials here + await databaseContextFactory.UseContextTaskReturn(context => + { + context.RepositorySettings.Attach(currentModel); + currentModel.AccessUser = null; + currentModel.AccessToken = null; + return context.Save(CancellationToken.None); // DCT: Must always run + }); + + throw; + } + + using (newRepo) + await newRepo.CheckoutObject( + oldSha, + currentModel.AccessUser, + currentModel.AccessToken, + false, + oldReference != null, + progressReporter.CreateSection("Checking out previous Detached Commit", 0.1), + cancellationToken); + } } } diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 5d4d686aaf2..735b9efa839 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -223,6 +223,42 @@ await jobManager.RegisterOperation( return Accepted(api); } + /// + /// Delete the repository. + /// + /// The for the operation. + /// A resulting in the of the operation. + /// Job to delete the repository created successfully. + /// The database entity for the requested instance could not be retrieved. The instance was likely detached. + [HttpPatch] + [TgsAuthorize(RepositoryRights.Reclone)] + [ProducesResponseType(typeof(RepositoryResponse), 202)] + [ProducesResponseType(typeof(ErrorMessageResponse), 410)] + public async ValueTask Reclone(CancellationToken cancellationToken) + { + var currentModel = await DatabaseContext + .RepositorySettings + .AsQueryable() + .Where(x => x.InstanceId == Instance.Id) + .FirstOrDefaultAsync(cancellationToken); + + if (currentModel == default) + return this.Gone(); + + Logger.LogInformation("Instance {instanceId} repository reclone initiated by user {userId}", Instance.Id, AuthenticationContext.User.Require(x => x.Id)); + + var repositoryUpdater = CreateRepositoryUpdateService(currentModel); + + var job = Job.Create(JobCode.RepositoryReclone, AuthenticationContext.User, Instance); + var api = currentModel.ToApi(); + await jobManager.RegisterOperation( + job, + (core, databaseContextFactory, paramJob, progressReporter, ct) => repositoryUpdater.RepositoryRecloneJob(core, databaseContextFactory, progressReporter, ct), + cancellationToken); + api.ActiveJob = job.ToApi(); + return Accepted(api); + } + /// /// Get the repository's status. /// @@ -443,17 +479,12 @@ bool CheckModified(Expression var job = Job.Create(JobCode.RepositoryUpdate, AuthenticationContext.User, Instance, RepositoryRights.CancelPendingChanges); job.Description = description; - var repositoryUpdater = new RepositoryUpdateService( - model, - currentModel, - AuthenticationContext.User, - loggerFactory.CreateLogger(), - Instance.Require(x => x.Id)); + var repositoryUpdater = CreateRepositoryUpdateService(currentModel); // Time to access git, do it in a job await jobManager.RegisterOperation( job, - repositoryUpdater.RepositoryUpdateJob, + (instance, databaseContextFactory, _, progressReporter, jobToken) => repositoryUpdater.RepositoryUpdateJob(model, instance, databaseContextFactory, progressReporter, jobToken), cancellationToken); api.ActiveJob = job.ToApi(); @@ -494,5 +525,17 @@ async ValueTask PopulateApi( cancellationToken); return needsDbUpdate; } + + /// + /// Creates a . + /// + /// The current . + /// A new . + RepositoryUpdateService CreateRepositoryUpdateService(RepositorySettings currentModel) + => new( + currentModel, + AuthenticationContext.User, + loggerFactory.CreateLogger(), + Instance.Require(x => x.Id)); } } diff --git a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs index 0d6ecce8648..86fe4ac9119 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs @@ -116,6 +116,8 @@ public async Task AbortLongCloneAndCloneSomethingQuick(Task longClo await ApiAssert.ThrowsException(() => Checkout(new RepositoryUpdateRequest { Reference = "master", CheckoutSha = "286bb75" }, false, false, cancellationToken), ErrorCode.RepoMismatchShaAndReference); var updated = await Checkout(new RepositoryUpdateRequest { CheckoutSha = "286bb75" }, false, false, cancellationToken); + await RecloneTest(cancellationToken); + // Fake SHA updated = await Checkout(new RepositoryUpdateRequest { CheckoutSha = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, true, false, cancellationToken); @@ -142,6 +144,23 @@ await repositoryClient.Update(new RepositoryUpdateRequest await TestMergeTests(updated, prNumber, cancellationToken); } + async ValueTask RecloneTest(CancellationToken cancellationToken) + { + var initialState = await repositoryClient.Read(cancellationToken); + Assert.IsNotNull(initialState.Reference); + Assert.IsNotNull(initialState.RevisionInformation); + Assert.IsNotNull(initialState.RevisionInformation.CommitSha); + Assert.IsNotNull(initialState.RevisionInformation.OriginCommitSha); + + var reclone = await repositoryClient.Reclone(cancellationToken); + await WaitForJob(reclone.ActiveJob, 70, false, null, cancellationToken); + + var newState = await repositoryClient.Read(cancellationToken); + Assert.AreEqual(initialState.Reference, newState.Reference); + Assert.AreEqual(initialState.RevisionInformation.CommitSha, newState.RevisionInformation.CommitSha); + Assert.AreEqual(initialState.RevisionInformation.OriginCommitSha, newState.RevisionInformation.OriginCommitSha); + } + async ValueTask Checkout(RepositoryUpdateRequest updated, bool expectFailure, bool isRef, CancellationToken cancellationToken) { var newRef = isRef ? updated.Reference : updated.CheckoutSha; diff --git a/tests/Tgstation.Server.Tests/TestRepository.cs b/tests/Tgstation.Server.Tests/TestRepository.cs index e73731de404..59b47839942 100644 --- a/tests/Tgstation.Server.Tests/TestRepository.cs +++ b/tests/Tgstation.Server.Tests/TestRepository.cs @@ -45,7 +45,7 @@ public async Task TestRepoParentLookup() () => { }); const string StartSha = "af4da8beb9f9b374b04a3cc4d65acca662e8cc1a"; - await repo.CheckoutObject(StartSha, null, null, true, new JobProgressReporter(Mock.Of>(), null, (stage, progress) => { }), CancellationToken.None); + await repo.CheckoutObject(StartSha, null, null, true, false, new JobProgressReporter(Mock.Of>(), null, (stage, progress) => { }), CancellationToken.None); Assert.AreEqual(Host.Components.Repository.Repository.NoReference, repo.Reference); From 5b4719c1285b7a7e1a0d0a8a8b17ea7b809573c9 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 10 Aug 2024 21:44:58 -0400 Subject: [PATCH 012/111] Fix a race condition in OpenDreamInstaller --- .../Components/Engine/OpenDreamInstaller.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index f99d70bfa48..608d3addb5f 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -157,6 +157,8 @@ public override async ValueTask DownloadVersion(EngineV { Logger.LogTrace("OD repo seems to already exist, attempting load and fetch..."); repo = await repositoryManager.LoadRepository(cancellationToken); + if (repo == null) + throw new JobException("Can't load OpenDream repository! Please delete cache from disk!"); await repo!.FetchOrigin( progressSection1, From 77a20cce750264c30500803f9a922a1d8a90d4ce Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 10 Aug 2024 23:56:25 -0400 Subject: [PATCH 013/111] Implement CPU/Memory sampling Closes #611 --- .../Models/Response/DreamDaemonResponse.cs | 12 ++++ .../Components/DreamDaemonClient.cs | 7 ++- .../Components/IDreamDaemonClient.cs | 3 +- .../Components/Session/SessionController.cs | 7 +++ .../Components/Watchdog/IWatchdog.cs | 8 +++ .../Components/Watchdog/WatchdogBase.cs | 11 ++++ .../Controllers/DreamDaemonController.cs | 28 +++++++-- .../System/IProcessBase.cs | 16 ++++- src/Tgstation.Server.Host/System/Process.cs | 30 ++++++++++ .../System/ProcessExecutor.cs | 11 ++++ .../System/TestPosixSignalHandler.cs | 1 + .../Live/Instance/DeploymentTest.cs | 2 +- .../Live/Instance/WatchdogTest.cs | 59 +++++++++++-------- .../Live/TestLiveServer.cs | 13 ++-- .../TestSystemInteraction.cs | 2 + tests/Tgstation.Server.Tests/TestVersions.cs | 1 + 16 files changed, 172 insertions(+), 39 deletions(-) diff --git a/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs b/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs index cad5094d1a2..8f54e19c626 100644 --- a/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs +++ b/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs @@ -59,5 +59,17 @@ public sealed class DreamDaemonResponse : DreamDaemonApiBase /// [ResponseOptions] public bool? CurrentAllowWebclient { get; set; } + + /// + /// The amount of RAM in use by the game server in bytes. + /// + [ResponseOptions] + public long? ImmediateMemoryUsage { get; set; } + + /// + /// The CPU usage of the game server on a scale from 0-1. + /// + [ResponseOptions] + public double? ImmediateCpuUsage { get; set; } } } diff --git a/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs b/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs index cf4eb3ea874..3d5e8a6786f 100644 --- a/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs +++ b/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs @@ -43,7 +43,12 @@ public DreamDaemonClient(IApiClient apiClient, Instance instance) public ValueTask Restart(CancellationToken cancellationToken) => apiClient.Patch(Routes.DreamDaemon, instance.Id!.Value, cancellationToken); /// - public ValueTask Read(CancellationToken cancellationToken) => apiClient.Read(Routes.DreamDaemon, instance.Id!.Value, cancellationToken); + public ValueTask Read(ulong? profileMs, CancellationToken cancellationToken) => apiClient.Read( + profileMs.HasValue + ? $"{Routes.DreamDaemon}?profileMs={profileMs.Value}" + : Routes.DreamDaemon, + instance.Id!.Value, + cancellationToken); /// public ValueTask Update(DreamDaemonRequest dreamDaemon, CancellationToken cancellationToken) => apiClient.Update(Routes.DreamDaemon, dreamDaemon ?? throw new ArgumentNullException(nameof(dreamDaemon)), instance.Id!.Value, cancellationToken); diff --git a/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs b/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs index 7a7a6ead7cf..50c416b97c1 100644 --- a/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs +++ b/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs @@ -14,9 +14,10 @@ public interface IDreamDaemonClient /// /// Get the represented by the . /// + /// The amount of time to spend performance profiling. /// The for the operation. /// A resulting in the information. - ValueTask Read(CancellationToken cancellationToken); + ValueTask Read(ulong? profileMs = null, CancellationToken cancellationToken = default); /// /// Start . diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 10556ca9fc2..ed1509cf564 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -121,6 +121,9 @@ async Task Wrap() /// public FifoSemaphore TopicSendSemaphore { get; } + /// + public long MemoryUsage => process.MemoryUsage; + /// /// The for the . /// @@ -513,6 +516,10 @@ public ValueTask CreateDump(string outputFile, bool minidump, CancellationToken return process.CreateDump(outputFile, minidump, cancellationToken); } + /// + public ValueTask GetCpuUsage(TimeSpan waitingWindow, CancellationToken cancellationToken) + => process.GetCpuUsage(waitingWindow, cancellationToken); + /// /// The for . /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs index 96c6c9ccf9a..1decbb915ac 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs @@ -102,5 +102,13 @@ public interface IWatchdog : IComponentService, IAsyncDisposable, IEventConsumer /// The for the operation. /// A resulting in if the broadcast succeeded., otherwise. ValueTask Broadcast(string message, CancellationToken cancellationToken); + + /// + /// Profile memory and CPU usage of the running game server. + /// + /// The duration to profile the CPU usage of the game server for. + /// The for the operation. + /// A resulting in the performance metrics or if the server is offline. + Task<(long MemoryUsage, double CpuUsage)?> PerformanceProfile(TimeSpan timeSpan, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 28156db6e5e..3a9b4029017 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -289,6 +289,17 @@ public async ValueTask ChangeSettings(DreamDaemonLaunchParameters launchPa return true; } + /// + public async Task<(long MemoryUsage, double CpuUsage)?> PerformanceProfile(TimeSpan timeSpan, CancellationToken cancellationToken) + { + var controller = GetActiveController(); + if (controller == null) + return null; + + var cpuUsage = await controller.GetCpuUsage(timeSpan, cancellationToken); + return (controller.MemoryUsage, CpuUsage: cpuUsage); + } + /// public async ValueTask HandleChatCommand(string commandName, string arguments, ChatUser sender, CancellationToken cancellationToken) { diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index d60fda967d5..3c9854a1a3f 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -98,6 +98,7 @@ await jobManager.RegisterOperation( /// /// Get the watchdog status. /// + /// The amount of time to spend profiling game server CPU performance. /// The for the operation. /// A resulting in the of the operation. /// Read information successfully. @@ -106,7 +107,7 @@ await jobManager.RegisterOperation( [TgsAuthorize(DreamDaemonRights.ReadMetadata | DreamDaemonRights.ReadRevision)] [ProducesResponseType(typeof(DreamDaemonResponse), 200)] [ProducesResponseType(typeof(ErrorMessageResponse), 410)] - public ValueTask Read(CancellationToken cancellationToken) => ReadImpl(null, false, cancellationToken); + public ValueTask Read([FromQuery] ulong? profileMs, CancellationToken cancellationToken) => ReadImpl(null, profileMs, false, cancellationToken); /// /// Stops the Watchdog if it's running. @@ -252,7 +253,7 @@ bool CheckModified(Expression - /// Implementation of . + /// Implementation of . /// /// The to operate on if any. + /// The amount of time to spend profiling game server CPU performance. /// If there was a settings change made that forced a switch to . /// The for the operation. /// A resulting in the of the operation. - ValueTask ReadImpl(DreamDaemonSettings? settings, bool knownForcedReboot, CancellationToken cancellationToken) +#pragma warning disable CA1502 // TODO: Decomplexify + ValueTask ReadImpl(DreamDaemonSettings? settings, ulong? profileMs, bool knownForcedReboot, CancellationToken cancellationToken) +#pragma warning restore CA1502 => WithComponentInstance(async instance => { var dd = instance.Watchdog; - var metadata = (AuthenticationContext.GetRight(RightsType.DreamDaemon) & (ulong)DreamDaemonRights.ReadMetadata) != 0; + + Task<(long MemoryUsage, double CpuUsage)?>? profilingTask = null; + if (metadata && profileMs.HasValue && dd.Status == WatchdogStatus.Online) + profilingTask = dd.PerformanceProfile(TimeSpan.FromMilliseconds(profileMs.Value), cancellationToken); + var revision = (AuthenticationContext.GetRight(RightsType.DreamDaemon) & (ulong)DreamDaemonRights.ReadRevision) != 0; if (settings == null) @@ -385,6 +393,16 @@ ValueTask ReadImpl(DreamDaemonSettings? settings, bool knownForce result.LogOutput = settings.LogOutput; result.MapThreads = settings.MapThreads; result.Minidumps = settings.Minidumps; + + if (profilingTask != null) + { + var profile = await profilingTask; + if (profile.HasValue) + { + result.ImmediateMemoryUsage = profile.Value.MemoryUsage; + result.ImmediateCpuUsage = profile.Value.CpuUsage; + } + } } if (revision) diff --git a/src/Tgstation.Server.Host/System/IProcessBase.cs b/src/Tgstation.Server.Host/System/IProcessBase.cs index 92f6bdb1590..f4a613b62c5 100644 --- a/src/Tgstation.Server.Host/System/IProcessBase.cs +++ b/src/Tgstation.Server.Host/System/IProcessBase.cs @@ -1,4 +1,5 @@ -using System.Threading; +using System; +using System.Threading; using System.Threading.Tasks; namespace Tgstation.Server.Host.System @@ -13,6 +14,19 @@ public interface IProcessBase /// Task Lifetime { get; } + /// + /// Gets the process' memory usage in bytes. + /// + long MemoryUsage { get; } + + /// + /// Measures the 's CPU use percentage over a period of time. + /// + /// The to measure the percentage over. + /// The for the operation. + /// A ranging from 0-1 representing the percentage of the process' CPU time that was measured. + ValueTask GetCpuUsage(TimeSpan waitingWindow, CancellationToken cancellationToken); + /// /// Set's the owned to a non-normal value. /// diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index 896195944c5..2224ee3aace 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -7,6 +7,7 @@ using Microsoft.Win32.SafeHandles; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.System { @@ -22,11 +23,19 @@ sealed class Process : IProcess /// public Task Lifetime { get; } + /// + public long MemoryUsage => handle.VirtualMemorySize64; + /// /// The for the . /// readonly IProcessFeatures processFeatures; + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + /// /// The for the . /// @@ -62,6 +71,7 @@ sealed class Process : IProcess /// Initializes a new instance of the class. /// /// The value of . + /// The value of . /// The value of . /// The override value of . /// The value of . @@ -69,6 +79,7 @@ sealed class Process : IProcess /// If was NOT just created. public Process( IProcessFeatures processFeatures, + IAsyncDelayer asyncDelayer, global::System.Diagnostics.Process handle, CancellationTokenSource? readerCts, Task? readTask, @@ -84,6 +95,7 @@ public Process( cancellationTokenSource = readerCts ?? new CancellationTokenSource(); this.processFeatures = processFeatures ?? throw new ArgumentNullException(nameof(processFeatures)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.readTask = readTask; @@ -233,6 +245,24 @@ public ValueTask CreateDump(string outputFile, bool minidump, CancellationToken return processFeatures.CreateDump(handle, outputFile, minidump, cancellationToken); } + /// + public async ValueTask GetCpuUsage(TimeSpan waitingWindow, CancellationToken cancellationToken) + { + var startCpuUsage = handle.TotalProcessorTime; + var stopwatch = Stopwatch.StartNew(); + await asyncDelayer.Delay(waitingWindow, cancellationToken); + + var endCpuUsage = handle.TotalProcessorTime; + var totalElapsedTime = stopwatch.Elapsed; + + var cpuUsedMs = (endCpuUsage - startCpuUsage).TotalMilliseconds; + var totalMsPassed = totalElapsedTime.TotalMilliseconds; + + var cpuUsageTotal = cpuUsedMs / (Environment.ProcessorCount * totalMsPassed); + + return cpuUsageTotal; + } + /// /// Attaches a log message to the process' exit event. /// diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 01fe2c00a48..67d77d36006 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.System { @@ -32,6 +33,11 @@ sealed class ProcessExecutor : IProcessExecutor /// readonly IIOManager ioManager; + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + /// /// The for the . /// @@ -64,16 +70,19 @@ public static void WithProcessLaunchExclusivity(Action action) /// /// The value of . /// The value of . + /// The value of . /// The value of . /// The value of . public ProcessExecutor( IProcessFeatures processFeatures, IIOManager ioManager, + IAsyncDelayer asyncDelayer, ILogger logger, ILoggerFactory loggerFactory) { this.processFeatures = processFeatures ?? throw new ArgumentNullException(nameof(processFeatures)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); } @@ -195,6 +204,7 @@ public async ValueTask LaunchProcess( var process = new Process( processFeatures, + asyncDelayer, handle, disposeCts, readTask, @@ -354,6 +364,7 @@ Process CreateFromExistingHandle(global::System.Diagnostics.Process handle) var pid = handle.Id; return new Process( processFeatures, + asyncDelayer, handle, null, null, diff --git a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs index dd0f5fd5fb2..6b01eec4f09 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs @@ -61,6 +61,7 @@ public async Task TestSignalListening() new DefaultIOManager(), loggerFactory.CreateLogger()), Mock.Of(), + Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); await using var subProc = await processExecutor diff --git a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs index 1b20cd560c3..ddab5d3aef9 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs @@ -230,7 +230,7 @@ await dreamMakerClient.Update(new DreamMakerRequest async Task TestVisibilityPermission(CancellationToken cancellationToken) { - var updatedDD = await dreamDaemonClient.Read(cancellationToken); + var updatedDD = await dreamDaemonClient.Read(null, cancellationToken); Assert.AreEqual(DreamDaemonVisibility.Public, updatedDD.Visibility); updatedDD = await dreamDaemonClient.Update(new DreamDaemonRequest { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 26bba79305f..e4b53d03073 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -150,6 +150,8 @@ async Task UpdateDDSettings() }, cancellationToken); Assert.AreEqual(47, updated.OpenDreamTopicPort); + Assert.IsFalse(updated.ImmediateCpuUsage.HasValue); + Assert.IsFalse(updated.ImmediateMemoryUsage.HasValue); } catch (ConflictException ex) when (ex.ErrorCode == ErrorCode.PortNotAvailable) { @@ -266,7 +268,7 @@ async ValueTask RunTest(bool useTrusted) do { await Task.Delay(TimeSpan.FromSeconds(1), tempToken); - currentStatus = await instanceClient.DreamDaemon.Read(tempToken); + currentStatus = await instanceClient.DreamDaemon.Read(null, tempToken); } while (currentStatus.Status != WatchdogStatus.Offline); } @@ -358,7 +360,7 @@ await ApiAssert.ThrowsException(() => in // This one fucks with the access_identifer, run it in isolation await WhiteBoxValidateBridgeRequestLimitAndTestChunking(cancellationToken); - var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); + var ddInfo = await instanceClient.DreamDaemon.Read(null, cancellationToken); await CheckDMApiFail(ddInfo.ActiveCompileJob, cancellationToken); var deleteJob = await deleteJobTask; @@ -613,7 +615,7 @@ async Task TestDMApiFreeDeploy(CancellationToken cancellationToken) await WaitForJob(startJob, 40, false, null, cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); ValidateSessionId(daemonStatus, true); @@ -714,17 +716,19 @@ async Task RunBasicTest(CancellationToken cancellationToken) await WaitForJob(startJob, 40, false, null, cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); ValidateSessionId(daemonStatus, true); await CheckDDPriority(); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); + Assert.IsFalse(daemonStatus.ImmediateMemoryUsage.HasValue); + Assert.IsFalse(daemonStatus.ImmediateCpuUsage.HasValue); await GracefulWatchdogShutdown(cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); Assert.IsFalse(daemonStatus.SessionId.HasValue); await ExpectGameDirectoryCount(1, cancellationToken); @@ -845,6 +849,7 @@ async Task RunHealthCheckTest(bool checkDump, CancellationToken cancellationToke ? new WindowsProcessFeatures(Mock.Of>()) : new PosixProcessFeatures(new Lazy(() => executor), new DefaultIOManager(), Mock.Of>()), Mock.Of(), + Mock.Of(), Mock.Of>(), LoggerFactory.Create(x => { })); await using var ourProcessHandler = executor @@ -894,7 +899,7 @@ async Task RunHealthCheckTest(bool checkDump, CancellationToken cancellationToke var timeout = 20; do { - ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + ddStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(1U, ddStatus.HealthCheckSeconds.Value); if (ddStatus.Status.Value == WatchdogStatus.Offline) { @@ -1114,11 +1119,11 @@ ushort FindTopicPort() // - Injects a custom bridge handler into the bridge registrar and makes the test hack into the DMAPI and change its access_identifier async Task WhiteBoxChatCommandTest(CancellationToken cancellationToken) { - var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); + var ddInfo = await instanceClient.DreamDaemon.Read(null, cancellationToken); for (int i = 0; ddInfo.Status != WatchdogStatus.Online && i < 15; ++i) { await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); - ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); + ddInfo = await instanceClient.DreamDaemon.Read(null, cancellationToken); } Assert.AreEqual(WatchdogStatus.Online, ddInfo.Status); @@ -1169,7 +1174,7 @@ async Task WhiteBoxChatCommandTest(CancellationToken cancellationToken) var endTime = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5); - ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); + ddInfo = await instanceClient.DreamDaemon.Read(null, cancellationToken); await CheckDMApiFail(ddInfo.ActiveCompileJob, cancellationToken); CheckEmbedsTest(embedsResponse, startTime, endTime); @@ -1270,7 +1275,7 @@ async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken) await instanceClient.DreamDaemon.Shutdown(cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); } @@ -1317,7 +1322,7 @@ async Task RunLongRunningTestThenUpdateWithNewDme(CancellationToken cancellation await instanceClient.DreamDaemon.Shutdown(cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); } @@ -1330,7 +1335,7 @@ async Task RunLongRunningTestThenUpdateWithByondVersionSwitch(CancellationToken var currentByondVersion = await instanceClient.Engine.ActiveVersion(cancellationToken); Assert.AreNotEqual(versionToInstall, currentByondVersion.EngineVersion); - var initialStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var initialStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); var startJob = await StartDD(cancellationToken); @@ -1360,11 +1365,17 @@ async Task RunLongRunningTestThenUpdateWithByondVersionSwitch(CancellationToken await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, true, cancellationToken); - var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var daemonStatus = await instanceClient.DreamDaemon.Read(500, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); Assert.IsNotNull(daemonStatus.ActiveCompileJob); ValidateSessionId(daemonStatus, true); + Assert.IsTrue(daemonStatus.ImmediateCpuUsage.HasValue); + Assert.IsTrue(daemonStatus.ImmediateMemoryUsage.HasValue); + + // Assert.AreNotEqual(0, daemonStatus.ImmediateCpuUsage.Value); sleep_offline cucks this check + Assert.AreNotEqual(0, daemonStatus.ImmediateMemoryUsage.Value); + Assert.AreEqual(initialStatus.ActiveCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; Assert.AreNotEqual(daemonStatus.ActiveCompileJob.EngineVersion, newerCompileJob.EngineVersion); @@ -1383,7 +1394,7 @@ async Task RunLongRunningTestThenUpdateWithByondVersionSwitch(CancellationToken await instanceClient.DreamDaemon.Shutdown(cancellationToken); await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); } @@ -1416,7 +1427,7 @@ public async Task StartAndLeaveRunning(CancellationToken cancellationToken) KillDD(firstTime); firstTime = false; await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); } while (daemonStatus.Status == WatchdogStatus.Online); Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status); @@ -1425,7 +1436,7 @@ public async Task StartAndLeaveRunning(CancellationToken cancellationToken) do { KillDD(false); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); } while (daemonStatus.Status == WatchdogStatus.Online || daemonStatus.Status == WatchdogStatus.Restoring); Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status); @@ -1460,7 +1471,7 @@ public Task TellWorldToReboot(bool waitForOnlineIfRestoring => TellWorldToReboot2(instanceClient, instanceManager, topicClient, FindTopicPort(), waitForOnlineIfRestoring || testVersion.Engine.Value == EngineType.OpenDream, cancellationToken, source); public static async Task TellWorldToReboot2(IInstanceClient instanceClient, IInstanceManager instanceManager, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken, [CallerLineNumber]int source = 0, [CallerFilePath]string path = null) { - var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.IsNotNull(daemonStatus.StagedCompileJob); var initialSession = daemonStatus.ActiveCompileJob; @@ -1478,7 +1489,7 @@ public static async Task TellWorldToReboot2(IInstanceClient do { await Task.Delay(TimeSpan.FromSeconds(1), tempToken); - daemonStatus = await instanceClient.DreamDaemon.Read(tempToken); + daemonStatus = await instanceClient.DreamDaemon.Read(null, tempToken); } while (initialSession.Id == daemonStatus.ActiveCompileJob.Id); } @@ -1488,7 +1499,7 @@ public static async Task TellWorldToReboot2(IInstanceClient do { await Task.Delay(TimeSpan.FromSeconds(1), tempToken); - daemonStatus = await instanceClient.DreamDaemon.Read(tempToken); + daemonStatus = await instanceClient.DreamDaemon.Read(null, tempToken); } while (daemonStatus.Status == WatchdogStatus.Restoring); } @@ -1531,7 +1542,7 @@ await instanceClient.DreamMaker.Update(new DreamMakerRequest for (var i = 0; i < 10; ++i) await Task.Yield(); - var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); + var ddInfo = await instanceClient.DreamDaemon.Read(null, cancellationToken); var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob; Assert.IsNotNull(targetJob); if (requireApi) @@ -1549,14 +1560,14 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest SoftShutdown = true }, cancellationToken); - var newStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var newStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.IsTrue(newStatus.SoftShutdown.Value || newStatus.Status.Value == WatchdogStatus.Offline); var timeout = 40; do { await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); - var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var ddStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); if (ddStatus.Status.Value == WatchdogStatus.Offline) break; @@ -1584,7 +1595,7 @@ async Task CheckDMApiFail(CompileJobResponse compileJob, CancellationToken cance if (!checkLogs) return; - var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); if (daemonStatus.Status != WatchdogStatus.Offline || !daemonStatus.LogOutput.Value) return; @@ -1607,7 +1618,7 @@ async ValueTask TestLegacyBridgeEndpoint(CancellationToken cancellationToken) cancellationToken); Assert.IsNotNull(result); Assert.AreEqual("all gucci", result.StringData); - await CheckDMApiFail((await instanceClient.DreamDaemon.Read(cancellationToken)).ActiveCompileJob, cancellationToken); + await CheckDMApiFail((await instanceClient.DreamDaemon.Read(null, cancellationToken)).ActiveCompileJob, cancellationToken); } } } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index b34b14f066d..87383f216b4 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1095,6 +1095,7 @@ await ioManager.CopyDirectory( ? new WindowsProcessFeatures(loggerFactory.CreateLogger()) : new PosixProcessFeatures(new Lazy(() => processExecutor), ioManager, loggerFactory.CreateLogger()), ioManager, + Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); @@ -1197,7 +1198,7 @@ async ValueTask RunGitCommand(string args) { await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); - var status = await instanceClient.DreamDaemon.Read(cancellationToken); + var status = await instanceClient.DreamDaemon.Read(null, cancellationToken); if (updated) { @@ -1545,7 +1546,7 @@ await FailFast( if (openDreamOnly) return; - var dd = await instanceClient.DreamDaemon.Read(cancellationToken); + var dd = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); Assert.IsNotNull(dd.StagedCompileJob); Assert.AreNotEqual(dd.StagedCompileJob.Id, dd.ActiveCompileJob.Id); @@ -1628,7 +1629,7 @@ await FailFast( await jrt.WaitForJob(job, 130, job.Description.Contains("Reconnect chat bot") ? null : false, null, cancellationToken); } - var dd = await instanceClient.DreamDaemon.Read(cancellationToken); + var dd = await instanceClient.DreamDaemon.Read(500, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); Assert.IsNotNull(dd.StagedCompileJob); Assert.AreNotEqual(dd.StagedCompileJob.Id, dd.ActiveCompileJob.Id); @@ -1712,7 +1713,7 @@ async Task WaitForInitialJobs(IInstanceClient instanceClient) var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); - var dd = await instanceClient.DreamDaemon.Read(cancellationToken); + var dd = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); @@ -1720,7 +1721,7 @@ async Task WaitForInitialJobs(IInstanceClient instanceClient) await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort.Value, server.UsingBasicWatchdog); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); - dd = await instanceClient.DreamDaemon.Read(cancellationToken); + dd = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(dd.StagedCompileJob.Job.Id, compileJob.Id); expectedCompileJobId = compileJob.Id.Value; @@ -1755,7 +1756,7 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); - var currentDD = await instanceClient.DreamDaemon.Read(cancellationToken); + var currentDD = await instanceClient.DreamDaemon.Read(null, cancellationToken); Assert.AreEqual(expectedCompileJobId, currentDD.ActiveCompileJob.Id.Value); Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); diff --git a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs index e7c8e289620..da66aa3b7a7 100644 --- a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs +++ b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs @@ -25,6 +25,7 @@ public async Task TestScriptExecutionWithStdRead() var processExecutor = new ProcessExecutor( Mock.Of(), new DefaultIOManager(), + Mock.Of(), Mock.Of>(), loggerFactory); @@ -53,6 +54,7 @@ public async Task TestScriptExecutionWithFileOutput() var processExecutor = new ProcessExecutor( Mock.Of(), new DefaultIOManager(), + Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index adadd4bfcb2..7afbd15a718 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -211,6 +211,7 @@ await CachingFileDownloader.InitializeByondVersion( new DefaultIOManager(), loggerFactory.CreateLogger()), Mock.Of(), + Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); From 79fcf9c117bfb89343ad95cfa422c675fc377412 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 10 Aug 2024 23:56:38 -0400 Subject: [PATCH 014/111] Version bump to 6.9.0 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index c947fc3c5c2..15258f13a1f 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 6.8.0 + 6.9.0 5.2.0 10.7.0 7.0.0 From 7edf66440fa7f580bc04639bd15c437e4ee7144f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 11 Aug 2024 07:20:27 -0400 Subject: [PATCH 015/111] Remove CPU tracking. It's a nothing stat on multi-core systems --- .../Models/Response/DreamDaemonResponse.cs | 6 -- .../Components/DreamDaemonClient.cs | 6 +- .../Components/IDreamDaemonClient.cs | 3 +- .../Components/Session/SessionController.cs | 4 -- .../Components/Watchdog/IWatchdog.cs | 13 ++--- .../Components/Watchdog/WatchdogBase.cs | 14 +---- .../Controllers/DreamDaemonController.cs | 26 ++------- .../System/IProcessBase.cs | 11 +--- src/Tgstation.Server.Host/System/Process.cs | 27 --------- .../System/ProcessExecutor.cs | 11 ---- .../System/TestPosixSignalHandler.cs | 1 - .../Live/Instance/DeploymentTest.cs | 2 +- .../Live/Instance/WatchdogTest.cs | 57 +++++++++---------- .../Live/TestLiveServer.cs | 13 ++--- .../TestSystemInteraction.cs | 2 - tests/Tgstation.Server.Tests/TestVersions.cs | 1 - 16 files changed, 50 insertions(+), 147 deletions(-) diff --git a/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs b/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs index 8f54e19c626..d0b86cdaa41 100644 --- a/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs +++ b/src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs @@ -65,11 +65,5 @@ public sealed class DreamDaemonResponse : DreamDaemonApiBase /// [ResponseOptions] public long? ImmediateMemoryUsage { get; set; } - - /// - /// The CPU usage of the game server on a scale from 0-1. - /// - [ResponseOptions] - public double? ImmediateCpuUsage { get; set; } } } diff --git a/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs b/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs index 3d5e8a6786f..1ea2dccb3b7 100644 --- a/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs +++ b/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs @@ -43,10 +43,8 @@ public DreamDaemonClient(IApiClient apiClient, Instance instance) public ValueTask Restart(CancellationToken cancellationToken) => apiClient.Patch(Routes.DreamDaemon, instance.Id!.Value, cancellationToken); /// - public ValueTask Read(ulong? profileMs, CancellationToken cancellationToken) => apiClient.Read( - profileMs.HasValue - ? $"{Routes.DreamDaemon}?profileMs={profileMs.Value}" - : Routes.DreamDaemon, + public ValueTask Read(CancellationToken cancellationToken) => apiClient.Read( + Routes.DreamDaemon, instance.Id!.Value, cancellationToken); diff --git a/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs b/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs index 50c416b97c1..823c8d6a0ca 100644 --- a/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs +++ b/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs @@ -14,10 +14,9 @@ public interface IDreamDaemonClient /// /// Get the represented by the . /// - /// The amount of time to spend performance profiling. /// The for the operation. /// A resulting in the information. - ValueTask Read(ulong? profileMs = null, CancellationToken cancellationToken = default); + ValueTask Read(CancellationToken cancellationToken = default); /// /// Start . diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index ed1509cf564..6880ed7f6d2 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -516,10 +516,6 @@ public ValueTask CreateDump(string outputFile, bool minidump, CancellationToken return process.CreateDump(outputFile, minidump, cancellationToken); } - /// - public ValueTask GetCpuUsage(TimeSpan waitingWindow, CancellationToken cancellationToken) - => process.GetCpuUsage(waitingWindow, cancellationToken); - /// /// The for . /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs index 1decbb915ac..2220c569327 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs @@ -24,6 +24,11 @@ public interface IWatchdog : IComponentService, IAsyncDisposable, IEventConsumer /// WatchdogStatus Status { get; } + /// + /// Gets the memory usage of the game server in bytes. + /// + long? MemoryUsage { get; } + /// /// If the alpha server is the active server. /// @@ -102,13 +107,5 @@ public interface IWatchdog : IComponentService, IAsyncDisposable, IEventConsumer /// The for the operation. /// A resulting in if the broadcast succeeded., otherwise. ValueTask Broadcast(string message, CancellationToken cancellationToken); - - /// - /// Profile memory and CPU usage of the running game server. - /// - /// The duration to profile the CPU usage of the game server for. - /// The for the operation. - /// A resulting in the performance metrics or if the server is offline. - Task<(long MemoryUsage, double CpuUsage)?> PerformanceProfile(TimeSpan timeSpan, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 3a9b4029017..5d9326ce7dd 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -50,6 +50,9 @@ protected set } } + /// + public long? MemoryUsage => GetActiveController()?.MemoryUsage; + /// public abstract bool AlphaIsActive { get; } @@ -289,17 +292,6 @@ public async ValueTask ChangeSettings(DreamDaemonLaunchParameters launchPa return true; } - /// - public async Task<(long MemoryUsage, double CpuUsage)?> PerformanceProfile(TimeSpan timeSpan, CancellationToken cancellationToken) - { - var controller = GetActiveController(); - if (controller == null) - return null; - - var cpuUsage = await controller.GetCpuUsage(timeSpan, cancellationToken); - return (controller.MemoryUsage, CpuUsage: cpuUsage); - } - /// public async ValueTask HandleChatCommand(string commandName, string arguments, ChatUser sender, CancellationToken cancellationToken) { diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index 3c9854a1a3f..79e03ebb845 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -98,7 +98,6 @@ await jobManager.RegisterOperation( /// /// Get the watchdog status. /// - /// The amount of time to spend profiling game server CPU performance. /// The for the operation. /// A resulting in the of the operation. /// Read information successfully. @@ -107,7 +106,7 @@ await jobManager.RegisterOperation( [TgsAuthorize(DreamDaemonRights.ReadMetadata | DreamDaemonRights.ReadRevision)] [ProducesResponseType(typeof(DreamDaemonResponse), 200)] [ProducesResponseType(typeof(ErrorMessageResponse), 410)] - public ValueTask Read([FromQuery] ulong? profileMs, CancellationToken cancellationToken) => ReadImpl(null, profileMs, false, cancellationToken); + public ValueTask Read(CancellationToken cancellationToken) => ReadImpl(null, false, cancellationToken); /// /// Stops the Watchdog if it's running. @@ -253,7 +252,7 @@ bool CheckModified(Expression - /// Implementation of . + /// Implementation of . /// /// The to operate on if any. - /// The amount of time to spend profiling game server CPU performance. /// If there was a settings change made that forced a switch to . /// The for the operation. /// A resulting in the of the operation. #pragma warning disable CA1502 // TODO: Decomplexify - ValueTask ReadImpl(DreamDaemonSettings? settings, ulong? profileMs, bool knownForcedReboot, CancellationToken cancellationToken) + ValueTask ReadImpl(DreamDaemonSettings? settings, bool knownForcedReboot, CancellationToken cancellationToken) #pragma warning restore CA1502 => WithComponentInstance(async instance => { var dd = instance.Watchdog; var metadata = (AuthenticationContext.GetRight(RightsType.DreamDaemon) & (ulong)DreamDaemonRights.ReadMetadata) != 0; - - Task<(long MemoryUsage, double CpuUsage)?>? profilingTask = null; - if (metadata && profileMs.HasValue && dd.Status == WatchdogStatus.Online) - profilingTask = dd.PerformanceProfile(TimeSpan.FromMilliseconds(profileMs.Value), cancellationToken); - var revision = (AuthenticationContext.GetRight(RightsType.DreamDaemon) & (ulong)DreamDaemonRights.ReadRevision) != 0; if (settings == null) @@ -380,6 +373,7 @@ ValueTask ReadImpl(DreamDaemonSettings? settings, ulong? profileM result.Visibility = settings.Visibility!.Value; result.SoftRestart = rstate == RebootState.Restart; result.SoftShutdown = rstate == RebootState.Shutdown; + result.ImmediateMemoryUsage = dd.MemoryUsage; if (rstate == RebootState.Normal && knownForcedReboot) result.SoftRestart = true; @@ -393,16 +387,6 @@ ValueTask ReadImpl(DreamDaemonSettings? settings, ulong? profileM result.LogOutput = settings.LogOutput; result.MapThreads = settings.MapThreads; result.Minidumps = settings.Minidumps; - - if (profilingTask != null) - { - var profile = await profilingTask; - if (profile.HasValue) - { - result.ImmediateMemoryUsage = profile.Value.MemoryUsage; - result.ImmediateCpuUsage = profile.Value.CpuUsage; - } - } } if (revision) diff --git a/src/Tgstation.Server.Host/System/IProcessBase.cs b/src/Tgstation.Server.Host/System/IProcessBase.cs index f4a613b62c5..f34d5e6597c 100644 --- a/src/Tgstation.Server.Host/System/IProcessBase.cs +++ b/src/Tgstation.Server.Host/System/IProcessBase.cs @@ -1,5 +1,4 @@ -using System; -using System.Threading; +using System.Threading; using System.Threading.Tasks; namespace Tgstation.Server.Host.System @@ -19,14 +18,6 @@ public interface IProcessBase /// long MemoryUsage { get; } - /// - /// Measures the 's CPU use percentage over a period of time. - /// - /// The to measure the percentage over. - /// The for the operation. - /// A ranging from 0-1 representing the percentage of the process' CPU time that was measured. - ValueTask GetCpuUsage(TimeSpan waitingWindow, CancellationToken cancellationToken); - /// /// Set's the owned to a non-normal value. /// diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index 2224ee3aace..64c92bf89b9 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -7,7 +7,6 @@ using Microsoft.Win32.SafeHandles; using Tgstation.Server.Host.IO; -using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.System { @@ -31,11 +30,6 @@ sealed class Process : IProcess /// readonly IProcessFeatures processFeatures; - /// - /// The for the . - /// - readonly IAsyncDelayer asyncDelayer; - /// /// The for the . /// @@ -71,7 +65,6 @@ sealed class Process : IProcess /// Initializes a new instance of the class. /// /// The value of . - /// The value of . /// The value of . /// The override value of . /// The value of . @@ -79,7 +72,6 @@ sealed class Process : IProcess /// If was NOT just created. public Process( IProcessFeatures processFeatures, - IAsyncDelayer asyncDelayer, global::System.Diagnostics.Process handle, CancellationTokenSource? readerCts, Task? readTask, @@ -95,7 +87,6 @@ public Process( cancellationTokenSource = readerCts ?? new CancellationTokenSource(); this.processFeatures = processFeatures ?? throw new ArgumentNullException(nameof(processFeatures)); - this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.readTask = readTask; @@ -245,24 +236,6 @@ public ValueTask CreateDump(string outputFile, bool minidump, CancellationToken return processFeatures.CreateDump(handle, outputFile, minidump, cancellationToken); } - /// - public async ValueTask GetCpuUsage(TimeSpan waitingWindow, CancellationToken cancellationToken) - { - var startCpuUsage = handle.TotalProcessorTime; - var stopwatch = Stopwatch.StartNew(); - await asyncDelayer.Delay(waitingWindow, cancellationToken); - - var endCpuUsage = handle.TotalProcessorTime; - var totalElapsedTime = stopwatch.Elapsed; - - var cpuUsedMs = (endCpuUsage - startCpuUsage).TotalMilliseconds; - var totalMsPassed = totalElapsedTime.TotalMilliseconds; - - var cpuUsageTotal = cpuUsedMs / (Environment.ProcessorCount * totalMsPassed); - - return cpuUsageTotal; - } - /// /// Attaches a log message to the process' exit event. /// diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 67d77d36006..01fe2c00a48 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -11,7 +11,6 @@ using Microsoft.Extensions.Logging; using Tgstation.Server.Host.IO; -using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.System { @@ -33,11 +32,6 @@ sealed class ProcessExecutor : IProcessExecutor /// readonly IIOManager ioManager; - /// - /// The for the . - /// - readonly IAsyncDelayer asyncDelayer; - /// /// The for the . /// @@ -70,19 +64,16 @@ public static void WithProcessLaunchExclusivity(Action action) /// /// The value of . /// The value of . - /// The value of . /// The value of . /// The value of . public ProcessExecutor( IProcessFeatures processFeatures, IIOManager ioManager, - IAsyncDelayer asyncDelayer, ILogger logger, ILoggerFactory loggerFactory) { this.processFeatures = processFeatures ?? throw new ArgumentNullException(nameof(processFeatures)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); - this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); } @@ -204,7 +195,6 @@ public async ValueTask LaunchProcess( var process = new Process( processFeatures, - asyncDelayer, handle, disposeCts, readTask, @@ -364,7 +354,6 @@ Process CreateFromExistingHandle(global::System.Diagnostics.Process handle) var pid = handle.Id; return new Process( processFeatures, - asyncDelayer, handle, null, null, diff --git a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs index 6b01eec4f09..dd0f5fd5fb2 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs @@ -61,7 +61,6 @@ public async Task TestSignalListening() new DefaultIOManager(), loggerFactory.CreateLogger()), Mock.Of(), - Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); await using var subProc = await processExecutor diff --git a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs index ddab5d3aef9..1b20cd560c3 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs @@ -230,7 +230,7 @@ await dreamMakerClient.Update(new DreamMakerRequest async Task TestVisibilityPermission(CancellationToken cancellationToken) { - var updatedDD = await dreamDaemonClient.Read(null, cancellationToken); + var updatedDD = await dreamDaemonClient.Read(cancellationToken); Assert.AreEqual(DreamDaemonVisibility.Public, updatedDD.Visibility); updatedDD = await dreamDaemonClient.Update(new DreamDaemonRequest { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index e4b53d03073..08f436b26df 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -150,7 +150,6 @@ async Task UpdateDDSettings() }, cancellationToken); Assert.AreEqual(47, updated.OpenDreamTopicPort); - Assert.IsFalse(updated.ImmediateCpuUsage.HasValue); Assert.IsFalse(updated.ImmediateMemoryUsage.HasValue); } catch (ConflictException ex) when (ex.ErrorCode == ErrorCode.PortNotAvailable) @@ -268,7 +267,7 @@ async ValueTask RunTest(bool useTrusted) do { await Task.Delay(TimeSpan.FromSeconds(1), tempToken); - currentStatus = await instanceClient.DreamDaemon.Read(null, tempToken); + currentStatus = await instanceClient.DreamDaemon.Read(tempToken); } while (currentStatus.Status != WatchdogStatus.Offline); } @@ -360,7 +359,7 @@ await ApiAssert.ThrowsException(() => in // This one fucks with the access_identifer, run it in isolation await WhiteBoxValidateBridgeRequestLimitAndTestChunking(cancellationToken); - var ddInfo = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); await CheckDMApiFail(ddInfo.ActiveCompileJob, cancellationToken); var deleteJob = await deleteJobTask; @@ -615,7 +614,7 @@ async Task TestDMApiFreeDeploy(CancellationToken cancellationToken) await WaitForJob(startJob, 40, false, null, cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); ValidateSessionId(daemonStatus, true); @@ -716,19 +715,19 @@ async Task RunBasicTest(CancellationToken cancellationToken) await WaitForJob(startJob, 40, false, null, cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); ValidateSessionId(daemonStatus, true); await CheckDDPriority(); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); - Assert.IsFalse(daemonStatus.ImmediateMemoryUsage.HasValue); - Assert.IsFalse(daemonStatus.ImmediateCpuUsage.HasValue); + Assert.IsTrue(daemonStatus.ImmediateMemoryUsage.HasValue); + Assert.AreNotEqual(0, daemonStatus.ImmediateMemoryUsage.Value); await GracefulWatchdogShutdown(cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); Assert.IsFalse(daemonStatus.SessionId.HasValue); await ExpectGameDirectoryCount(1, cancellationToken); @@ -849,7 +848,6 @@ async Task RunHealthCheckTest(bool checkDump, CancellationToken cancellationToke ? new WindowsProcessFeatures(Mock.Of>()) : new PosixProcessFeatures(new Lazy(() => executor), new DefaultIOManager(), Mock.Of>()), Mock.Of(), - Mock.Of(), Mock.Of>(), LoggerFactory.Create(x => { })); await using var ourProcessHandler = executor @@ -899,7 +897,7 @@ async Task RunHealthCheckTest(bool checkDump, CancellationToken cancellationToke var timeout = 20; do { - ddStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(1U, ddStatus.HealthCheckSeconds.Value); if (ddStatus.Status.Value == WatchdogStatus.Offline) { @@ -1119,11 +1117,11 @@ ushort FindTopicPort() // - Injects a custom bridge handler into the bridge registrar and makes the test hack into the DMAPI and change its access_identifier async Task WhiteBoxChatCommandTest(CancellationToken cancellationToken) { - var ddInfo = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); for (int i = 0; ddInfo.Status != WatchdogStatus.Online && i < 15; ++i) { await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); - ddInfo = await instanceClient.DreamDaemon.Read(null, cancellationToken); + ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); } Assert.AreEqual(WatchdogStatus.Online, ddInfo.Status); @@ -1174,7 +1172,7 @@ async Task WhiteBoxChatCommandTest(CancellationToken cancellationToken) var endTime = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5); - ddInfo = await instanceClient.DreamDaemon.Read(null, cancellationToken); + ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); await CheckDMApiFail(ddInfo.ActiveCompileJob, cancellationToken); CheckEmbedsTest(embedsResponse, startTime, endTime); @@ -1275,7 +1273,7 @@ async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken) await instanceClient.DreamDaemon.Shutdown(cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); } @@ -1322,7 +1320,7 @@ async Task RunLongRunningTestThenUpdateWithNewDme(CancellationToken cancellation await instanceClient.DreamDaemon.Shutdown(cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); } @@ -1335,7 +1333,7 @@ async Task RunLongRunningTestThenUpdateWithByondVersionSwitch(CancellationToken var currentByondVersion = await instanceClient.Engine.ActiveVersion(cancellationToken); Assert.AreNotEqual(versionToInstall, currentByondVersion.EngineVersion); - var initialStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var initialStatus = await instanceClient.DreamDaemon.Read(cancellationToken); var startJob = await StartDD(cancellationToken); @@ -1365,15 +1363,12 @@ async Task RunLongRunningTestThenUpdateWithByondVersionSwitch(CancellationToken await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, true, cancellationToken); - var daemonStatus = await instanceClient.DreamDaemon.Read(500, cancellationToken); + var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); Assert.IsNotNull(daemonStatus.ActiveCompileJob); ValidateSessionId(daemonStatus, true); - Assert.IsTrue(daemonStatus.ImmediateCpuUsage.HasValue); Assert.IsTrue(daemonStatus.ImmediateMemoryUsage.HasValue); - - // Assert.AreNotEqual(0, daemonStatus.ImmediateCpuUsage.Value); sleep_offline cucks this check Assert.AreNotEqual(0, daemonStatus.ImmediateMemoryUsage.Value); Assert.AreEqual(initialStatus.ActiveCompileJob.Id, daemonStatus.ActiveCompileJob.Id); @@ -1394,7 +1389,7 @@ async Task RunLongRunningTestThenUpdateWithByondVersionSwitch(CancellationToken await instanceClient.DreamDaemon.Shutdown(cancellationToken); await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); } @@ -1427,7 +1422,7 @@ public async Task StartAndLeaveRunning(CancellationToken cancellationToken) KillDD(firstTime); firstTime = false; await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); } while (daemonStatus.Status == WatchdogStatus.Online); Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status); @@ -1436,7 +1431,7 @@ public async Task StartAndLeaveRunning(CancellationToken cancellationToken) do { KillDD(false); - daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); } while (daemonStatus.Status == WatchdogStatus.Online || daemonStatus.Status == WatchdogStatus.Restoring); Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status); @@ -1471,7 +1466,7 @@ public Task TellWorldToReboot(bool waitForOnlineIfRestoring => TellWorldToReboot2(instanceClient, instanceManager, topicClient, FindTopicPort(), waitForOnlineIfRestoring || testVersion.Engine.Value == EngineType.OpenDream, cancellationToken, source); public static async Task TellWorldToReboot2(IInstanceClient instanceClient, IInstanceManager instanceManager, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken, [CallerLineNumber]int source = 0, [CallerFilePath]string path = null) { - var daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsNotNull(daemonStatus.StagedCompileJob); var initialSession = daemonStatus.ActiveCompileJob; @@ -1489,7 +1484,7 @@ public static async Task TellWorldToReboot2(IInstanceClient do { await Task.Delay(TimeSpan.FromSeconds(1), tempToken); - daemonStatus = await instanceClient.DreamDaemon.Read(null, tempToken); + daemonStatus = await instanceClient.DreamDaemon.Read(tempToken); } while (initialSession.Id == daemonStatus.ActiveCompileJob.Id); } @@ -1499,7 +1494,7 @@ public static async Task TellWorldToReboot2(IInstanceClient do { await Task.Delay(TimeSpan.FromSeconds(1), tempToken); - daemonStatus = await instanceClient.DreamDaemon.Read(null, tempToken); + daemonStatus = await instanceClient.DreamDaemon.Read(tempToken); } while (daemonStatus.Status == WatchdogStatus.Restoring); } @@ -1542,7 +1537,7 @@ await instanceClient.DreamMaker.Update(new DreamMakerRequest for (var i = 0; i < 10; ++i) await Task.Yield(); - var ddInfo = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob; Assert.IsNotNull(targetJob); if (requireApi) @@ -1560,14 +1555,14 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest SoftShutdown = true }, cancellationToken); - var newStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var newStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsTrue(newStatus.SoftShutdown.Value || newStatus.Status.Value == WatchdogStatus.Offline); var timeout = 40; do { await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); - var ddStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); if (ddStatus.Status.Value == WatchdogStatus.Offline) break; @@ -1595,7 +1590,7 @@ async Task CheckDMApiFail(CompileJobResponse compileJob, CancellationToken cance if (!checkLogs) return; - var daemonStatus = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); if (daemonStatus.Status != WatchdogStatus.Offline || !daemonStatus.LogOutput.Value) return; @@ -1618,7 +1613,7 @@ async ValueTask TestLegacyBridgeEndpoint(CancellationToken cancellationToken) cancellationToken); Assert.IsNotNull(result); Assert.AreEqual("all gucci", result.StringData); - await CheckDMApiFail((await instanceClient.DreamDaemon.Read(null, cancellationToken)).ActiveCompileJob, cancellationToken); + await CheckDMApiFail((await instanceClient.DreamDaemon.Read(cancellationToken)).ActiveCompileJob, cancellationToken); } } } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 87383f216b4..b34b14f066d 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1095,7 +1095,6 @@ await ioManager.CopyDirectory( ? new WindowsProcessFeatures(loggerFactory.CreateLogger()) : new PosixProcessFeatures(new Lazy(() => processExecutor), ioManager, loggerFactory.CreateLogger()), ioManager, - Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); @@ -1198,7 +1197,7 @@ async ValueTask RunGitCommand(string args) { await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); - var status = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var status = await instanceClient.DreamDaemon.Read(cancellationToken); if (updated) { @@ -1546,7 +1545,7 @@ await FailFast( if (openDreamOnly) return; - var dd = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var dd = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); Assert.IsNotNull(dd.StagedCompileJob); Assert.AreNotEqual(dd.StagedCompileJob.Id, dd.ActiveCompileJob.Id); @@ -1629,7 +1628,7 @@ await FailFast( await jrt.WaitForJob(job, 130, job.Description.Contains("Reconnect chat bot") ? null : false, null, cancellationToken); } - var dd = await instanceClient.DreamDaemon.Read(500, cancellationToken); + var dd = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); Assert.IsNotNull(dd.StagedCompileJob); Assert.AreNotEqual(dd.StagedCompileJob.Id, dd.ActiveCompileJob.Id); @@ -1713,7 +1712,7 @@ async Task WaitForInitialJobs(IInstanceClient instanceClient) var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); - var dd = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var dd = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); @@ -1721,7 +1720,7 @@ async Task WaitForInitialJobs(IInstanceClient instanceClient) await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort.Value, server.UsingBasicWatchdog); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); - dd = await instanceClient.DreamDaemon.Read(null, cancellationToken); + dd = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(dd.StagedCompileJob.Job.Id, compileJob.Id); expectedCompileJobId = compileJob.Id.Value; @@ -1756,7 +1755,7 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); - var currentDD = await instanceClient.DreamDaemon.Read(null, cancellationToken); + var currentDD = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(expectedCompileJobId, currentDD.ActiveCompileJob.Id.Value); Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); diff --git a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs index da66aa3b7a7..e7c8e289620 100644 --- a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs +++ b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs @@ -25,7 +25,6 @@ public async Task TestScriptExecutionWithStdRead() var processExecutor = new ProcessExecutor( Mock.Of(), new DefaultIOManager(), - Mock.Of(), Mock.Of>(), loggerFactory); @@ -54,7 +53,6 @@ public async Task TestScriptExecutionWithFileOutput() var processExecutor = new ProcessExecutor( Mock.Of(), new DefaultIOManager(), - Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index 7afbd15a718..adadd4bfcb2 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -211,7 +211,6 @@ await CachingFileDownloader.InitializeByondVersion( new DefaultIOManager(), loggerFactory.CreateLogger()), Mock.Of(), - Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); From 0116a2ae1d3fba7d2deb751dd13dfe08f1029da0 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 11 Aug 2024 07:49:47 -0400 Subject: [PATCH 016/111] Add spacestation13 fork sync --- .github/workflows/update-ss13-org-mirror.yml | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/update-ss13-org-mirror.yml diff --git a/.github/workflows/update-ss13-org-mirror.yml b/.github/workflows/update-ss13-org-mirror.yml new file mode 100644 index 00000000000..e8dabd987e4 --- /dev/null +++ b/.github/workflows/update-ss13-org-mirror.yml @@ -0,0 +1,51 @@ +name: 'Sync spacestation13/tgstation-server' + +on: + push: + branches: + - dev + tags: + - '*' + workflow_dispatch: + +env: + TGS_DOTNET_VERSION: 8 + TGS_DOTNET_QUALITY: ga + +jobs: + fork-sync: + name: Fork Sync + runs-on: ubuntu-latest + steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Generate App Token + run: | + dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV + rm ${{ runner.temp }}/installation_secret.txt + env: + TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} + + - name: Push to Spacestation13 Fork + run: | + git config user.name "tgstation-server-ci[bot]" + git config user.email "161980869+tgstation-server-ci[bot]@users.noreply.github.com" + git push "https://tgstation-server-ci:${{ env.INSTALLATION_TOKEN }}@github.com/spacestation13/tgstation-server" + git push --tags "https://tgstation-server-ci:${{ env.INSTALLATION_TOKEN }}@github.com/spacestation13/tgstation-server" From c0c286ab562d10c3d6f6b3279fe7326b6c8b19a6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 11 Aug 2024 07:50:08 -0400 Subject: [PATCH 017/111] Add missing env var to master merge --- .github/workflows/stable-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stable-merge.yml b/.github/workflows/stable-merge.yml index 143a6198059..2a9a6eb7260 100644 --- a/.github/workflows/stable-merge.yml +++ b/.github/workflows/stable-merge.yml @@ -8,7 +8,7 @@ on: env: TGS_DOTNET_VERSION: 8 - OD_MIN_COMPAT_DOTNET_VERSION: 7 + TGS_DOTNET_QUALITY: ga jobs: master-merge: From 3d5d2319af4f34ff3e47464f272c91f0f867e5d6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 11 Aug 2024 08:33:27 -0400 Subject: [PATCH 018/111] Hopefully fix SS13 repo sync --- .github/workflows/update-ss13-org-mirror.yml | 27 ++++++++++++++++++- .../Tgstation.Server.ReleaseNotes/Program.cs | 15 +++++++---- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.github/workflows/update-ss13-org-mirror.yml b/.github/workflows/update-ss13-org-mirror.yml index e8dabd987e4..c5db334d7f5 100644 --- a/.github/workflows/update-ss13-org-mirror.yml +++ b/.github/workflows/update-ss13-org-mirror.yml @@ -23,11 +23,36 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Checkout + - name: Build Checkout + uses: actions/checkout@v4 + with: + path: temp_workspace + + - name: Restore + run: | + cd temp_workspace + dotnet restore + + - name: Build ReleaseNotes + run: | + cd temp_workspace + dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Generate App Token + run: | + cd temp_workspace + dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} --spacestation13 + echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV + rm ${{ runner.temp }}/installation_secret.txt + env: + TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} + + - name: Main Checkout uses: actions/checkout@v4 with: fetch-depth: 0 fetch-tags: true + token: ${{ env.INSTALLATION_TOKEN }} - name: Restore run: dotnet restore diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index b3856534865..26247c8d012 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -135,7 +135,7 @@ static async Task Main(string[] args) return 454233; } - await GenerateAppCredentials(client, args[1]); + await GenerateAppCredentials(client, args[1], false); return await EnsureRelease(client); } @@ -171,7 +171,8 @@ static async Task Main(string[] args) return 33847; } - await GenerateAppCredentials(client, args[2]); + bool toSS13 = args.Length > 3 && args[3].Equals("--spacestation13", StringComparison.OrdinalIgnoreCase); + await GenerateAppCredentials(client, args[2], true); var token = client.Credentials.GetToken(); var destPath = args[1]; @@ -1640,7 +1641,7 @@ [optional blank line(s), stripped] return 0; } - static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string pemBase64) + static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string pemBase64, bool toSS13) { var pemBytes = Convert.FromBase64String(pemBase64); var pem = Encoding.UTF8.GetString(pemBytes); @@ -1665,7 +1666,11 @@ static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string gitHubClient.Credentials = new Credentials(jwtStr, AuthenticationType.Bearer); - var installation = await gitHubClient.GitHubApps.GetRepositoryInstallationForCurrent(RepoOwner, RepoName); + var installation = await gitHubClient.GitHubApps.GetRepositoryInstallationForCurrent( + toSS13 + ? "spacestation13" + : RepoOwner, + RepoName); var installToken = await gitHubClient.GitHubApps.CreateInstallationToken(installation.Id); gitHubClient.Credentials = new Credentials(installToken.Token); @@ -1673,7 +1678,7 @@ static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string static async ValueTask CICompletionCheck(GitHubClient gitHubClient, string currentSha, string pemBase64) { - await GenerateAppCredentials(gitHubClient, pemBase64); + await GenerateAppCredentials(gitHubClient, pemBase64, false); await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Completion", currentSha) { From 918687d57fc00073295d1072741502e9a15aa5f8 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 11 Aug 2024 09:56:37 -0400 Subject: [PATCH 019/111] Fix the homepage logo not working on Windows after updating --- .../Controllers/RootController.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/RootController.cs b/src/Tgstation.Server.Host/Controllers/RootController.cs index df363970bb3..577fecf1c64 100644 --- a/src/Tgstation.Server.Host/Controllers/RootController.cs +++ b/src/Tgstation.Server.Host/Controllers/RootController.cs @@ -152,11 +152,17 @@ public IActionResult Index() [HttpGet("logo.svg")] public IActionResult GetLogo() { - var logoFileName = platformIdentifier.IsWindows // these are different because of motherfucking line endings -_- - ? LogoSvgWindowsName - : LogoSvgLinuxName; + // these are different because of motherfucking line endings -_- + if (platformIdentifier.IsWindows) + { + VirtualFileResult? result = this.TryServeFile(hostEnvironment, logger, $"{LogoSvgWindowsName}.svg"); + if (result != null) + return result; + + // BUT THE UPDATE PACKAGES ARE BUILT ON LINUX RAAAAAGH + } - return (IActionResult?)this.TryServeFile(hostEnvironment, logger, $"{logoFileName}.svg") ?? NotFound(); + return (IActionResult?)this.TryServeFile(hostEnvironment, logger, $"{LogoSvgLinuxName}.svg") ?? NotFound(); } /// From 54882037643bb2104ce07fedd582863fc7d3f45c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 11 Aug 2024 16:13:26 -0400 Subject: [PATCH 020/111] Fix ReleaseNotes build [APIDeploy][DMDeploy] --- tools/Tgstation.Server.ReleaseNotes/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index 26247c8d012..d9f5981df8c 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -172,7 +172,7 @@ static async Task Main(string[] args) } bool toSS13 = args.Length > 3 && args[3].Equals("--spacestation13", StringComparison.OrdinalIgnoreCase); - await GenerateAppCredentials(client, args[2], true); + await GenerateAppCredentials(client, args[2], toSS13); var token = client.Credentials.GetToken(); var destPath = args[1]; From a42dafb3ab831e1f3ae54b592d609d978669d3d5 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 11 Aug 2024 17:01:05 -0400 Subject: [PATCH 021/111] Add support for GitHub App installation authentication to `IGitHubClientFactory` --- .../Remote/GitHubRemoteDeploymentManager.cs | 12 +- .../Repository/GitHubRemoteFeatures.cs | 4 +- .../Controllers/AdministrationController.cs | 2 +- .../Core/ServerUpdater.cs | 2 +- .../Security/OAuth/GitHubOAuthValidator.cs | 4 +- .../Utils/GitHub/GitHubClientFactory.cs | 131 ++++++++++++++++-- .../Utils/GitHub/GitHubServiceFactory.cs | 13 +- .../Utils/GitHub/IGitHubClientFactory.cs | 20 ++- .../Utils/GitHub/IGitHubServiceFactory.cs | 15 +- .../Utils/GitHub/TestGitHubClientFactory.cs | 19 +-- .../Utils/GitHub/TestGitHubServiceFactory.cs | 16 ++- .../Live/DummyGitHubServiceFactory.cs | 8 +- .../Live/TestingGitHubService.cs | 2 +- 13 files changed, 190 insertions(+), 58 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs index 4e37e7d008b..9a04c0ac94e 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs @@ -79,13 +79,13 @@ await databaseContextFactory.UseContext( IGitHubService gitHubService; if (instanceAuthenticated) { - authenticatedGitHubService = gitHubServiceFactory.CreateService(repositorySettings.AccessToken!); + authenticatedGitHubService = await gitHubServiceFactory.CreateService(repositorySettings.AccessToken!, cancellationToken); gitHubService = authenticatedGitHubService; } else { authenticatedGitHubService = null; - gitHubService = gitHubServiceFactory.CreateService(); + gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); } var repoOwner = remoteInformation.RemoteRepositoryOwner!; @@ -175,8 +175,8 @@ public override async ValueTask> RemoveMergedTest } var gitHubService = repositorySettings.AccessToken != null - ? gitHubServiceFactory.CreateService(repositorySettings.AccessToken) - : gitHubServiceFactory.CreateService(); + ? await gitHubServiceFactory.CreateService(repositorySettings.AccessToken, cancellationToken) + : await gitHubServiceFactory.CreateService(cancellationToken); var tasks = revisionInformation .ActiveTestMerges @@ -255,7 +255,7 @@ protected override async ValueTask CommentOnTestMergeSource( int testMergeNumber, CancellationToken cancellationToken) { - var gitHubService = gitHubServiceFactory.CreateService(repositorySettings.AccessToken!); + var gitHubService = await gitHubServiceFactory.CreateService(repositorySettings.AccessToken!, cancellationToken); try { @@ -343,7 +343,7 @@ await databaseContextFactory.UseContext( return; } - var gitHubService = gitHubServiceFactory.CreateService(gitHubAccessToken); + var gitHubService = await gitHubServiceFactory.CreateService(gitHubAccessToken, cancellationToken); try { diff --git a/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs b/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs index 6109f490bb7..69289bb5d40 100644 --- a/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs +++ b/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs @@ -49,8 +49,8 @@ public GitHubRemoteFeatures(IGitHubServiceFactory gitHubServiceFactory, ILogger< CancellationToken cancellationToken) { var gitHubService = repositorySettings.AccessToken != null - ? gitHubServiceFactory.CreateService(repositorySettings.AccessToken) - : gitHubServiceFactory.CreateService(); + ? await gitHubServiceFactory.CreateService(repositorySettings.AccessToken, cancellationToken) + : await gitHubServiceFactory.CreateService(cancellationToken); PullRequest? pr = null; ApiException? exception = null; diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index d4c1335df77..c41113069d2 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -165,7 +165,7 @@ async Task CacheFactory() Uri? repoUrl = null; try { - var gitHubService = gitHubServiceFactory.CreateService(); + var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken); var releases = await gitHubService.GetTgsReleases(cancellationToken); diff --git a/src/Tgstation.Server.Host/Core/ServerUpdater.cs b/src/Tgstation.Server.Host/Core/ServerUpdater.cs index 8c9d9681687..1170537a12d 100644 --- a/src/Tgstation.Server.Host/Core/ServerUpdater.cs +++ b/src/Tgstation.Server.Host/Core/ServerUpdater.cs @@ -288,7 +288,7 @@ async ValueTask BeginUpdateImpl( { logger.LogDebug("Looking for GitHub releases version {version}...", newVersion); - var gitHubService = gitHubServiceFactory.CreateService(); + var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); var releases = await gitHubService.GetTgsReleases(cancellationToken); foreach (var kvp in releases) { diff --git a/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs b/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs index 3f3180f9da2..d535d978843 100644 --- a/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs +++ b/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs @@ -60,12 +60,12 @@ public GitHubOAuthValidator( { logger.LogTrace("Validating response code..."); - var gitHubService = gitHubServiceFactory.CreateService(); + var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken); var token = await gitHubService.CreateOAuthAccessToken(oAuthConfiguration, code, cancellationToken); if (token == null) return null; - var authenticatedClient = gitHubServiceFactory.CreateService(token); + var authenticatedClient = await gitHubServiceFactory.CreateService(token, cancellationToken); logger.LogTrace("Getting user details..."); var userId = await authenticatedClient.GetCurrentUserId(cancellationToken); diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs index e479388b79b..4cf31a5d289 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs @@ -1,9 +1,16 @@ using System; using System.Collections.Generic; +using System.IdentityModel.Tokens.Jwt; using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; + using Octokit; using Tgstation.Server.Host.Configuration; @@ -12,7 +19,7 @@ namespace Tgstation.Server.Host.Utils.GitHub { /// - sealed class GitHubClientFactory : IGitHubClientFactory + sealed class GitHubClientFactory : IGitHubClientFactory, IDisposable { /// /// Limit to the amount of days a can live in the . @@ -45,6 +52,11 @@ sealed class GitHubClientFactory : IGitHubClientFactory /// readonly Dictionary clientCache; + /// + /// The used to guard access to . + /// + readonly SemaphoreSlim clientCacheSemaphore; + /// /// Initializes a new instance of the class. /// @@ -61,50 +73,139 @@ public GitHubClientFactory( generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); clientCache = new Dictionary(); + clientCacheSemaphore = new SemaphoreSlim(1, 1); } /// - public IGitHubClient CreateClient() => GetOrCreateClient(generalConfiguration.GitHubAccessToken); + public void Dispose() => clientCacheSemaphore.Dispose(); + + /// + public async ValueTask CreateClient(CancellationToken cancellationToken) + => (await GetOrCreateClient( + generalConfiguration.GitHubAccessToken, + null, + cancellationToken))!; /// - public IGitHubClient CreateClient(string accessToken) - => GetOrCreateClient( - accessToken ?? throw new ArgumentNullException(nameof(accessToken))); + public async ValueTask CreateClient(string accessToken, CancellationToken cancellationToken) + => (await GetOrCreateClient( + accessToken ?? throw new ArgumentNullException(nameof(accessToken)), + null, + cancellationToken))!; + + /// + public ValueTask CreateInstallationClient(string serializedPem, long repositoryId, CancellationToken cancellationToken) + => GetOrCreateClient(serializedPem, repositoryId, cancellationToken); /// - /// Retrieve a from the or add a new one based on a given . + /// Retrieve a from the or add a new one based on a given . /// - /// Optional access token to use as credentials. - /// The for the given . - GitHubClient GetOrCreateClient(string? accessToken) + /// Optional access token to use as credentials or GitHub App private key. If using a private key, must be set. + /// Setting this specifies is a private key and a GitHub App installation authenticated client will be returned. + /// The for the operation. + /// A resulting in the for the given or if authentication failed. +#pragma warning disable CA1506 // TODO: Decomplexify + async ValueTask GetOrCreateClient(string? accessTokenOrSerializedPem, long? installationRepositoryId, CancellationToken cancellationToken) +#pragma warning restore CA1506 { GitHubClient client; bool cacheHit; DateTimeOffset? lastUsed; - lock (clientCache) + using (await SemaphoreSlimContext.Lock(clientCacheSemaphore, cancellationToken)) { string cacheKey; - if (String.IsNullOrWhiteSpace(accessToken)) + if (String.IsNullOrWhiteSpace(accessTokenOrSerializedPem)) { - accessToken = null; + accessTokenOrSerializedPem = null; cacheKey = DefaultCacheKey; } else - cacheKey = accessToken; + cacheKey = accessTokenOrSerializedPem; cacheHit = clientCache.TryGetValue(cacheKey, out var tuple); var now = DateTimeOffset.UtcNow; if (!cacheHit) { + logger.LogTrace("Creating new GitHubClient..."); var product = assemblyInformationProvider.ProductInfoHeaderValue.Product!; client = new GitHubClient( new ProductHeaderValue( product.Name, product.Version)); - if (accessToken != null) - client.Credentials = new Credentials(accessToken); + if (accessTokenOrSerializedPem != null) + { + if (installationRepositoryId.HasValue) + { + logger.LogTrace("Performing GitHub App authentication for installation on repository {installationRepositoryId}", installationRepositoryId.Value); + var splits = accessTokenOrSerializedPem.Split(':'); + if (splits.Length != 2) + { + logger.LogError("Failed to parse serialized Client ID & PEM! Expected 2 chunks, got {chunkCount}", splits.Length); + return null; + } + + byte[] pemBytes; + try + { + pemBytes = Convert.FromBase64String(splits[1]); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to parse supposed base64 PEM!"); + return null; + } + + var pem = Encoding.UTF8.GetString(pemBytes); + + using var rsa = RSA.Create(); + rsa.ImportFromPem(pem); + + var signingCredentials = new SigningCredentials(new RsaSecurityKey(rsa), SecurityAlgorithms.RsaSha256); + var jwtSecurityTokenHandler = new JwtSecurityTokenHandler { SetDefaultTimesOnTokenCreation = false }; + + var nowDateTime = DateTime.UtcNow; + + var jwt = jwtSecurityTokenHandler.CreateToken(new SecurityTokenDescriptor + { + Issuer = splits[0], + Expires = nowDateTime.AddMinutes(10), + IssuedAt = nowDateTime, + SigningCredentials = signingCredentials, + }); + + var jwtStr = jwtSecurityTokenHandler.WriteToken(jwt); + + client.Credentials = new Credentials(jwtStr, AuthenticationType.Bearer); + + Installation installation; + try + { + installation = await client.GitHubApps.GetRepositoryInstallationForCurrent(installationRepositoryId.Value); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to perform app authentication!"); + return null; + } + + cancellationToken.ThrowIfCancellationRequested(); + try + { + var installToken = await client.GitHubApps.CreateInstallationToken(installation.Id); + + client.Credentials = new Credentials(installToken.Token); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to perform installation authentication!"); + return null; + } + } + else + client.Credentials = new Credentials(accessTokenOrSerializedPem); + } clientCache.Add(cacheKey, (Client: client, LastUsed: now)); lastUsed = null; diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs index efa08d13bba..a38b84b271d 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs @@ -1,4 +1,6 @@ using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -44,13 +46,16 @@ public GitHubServiceFactory( } /// - public IGitHubService CreateService() => CreateServiceImpl(gitHubClientFactory.CreateClient()); + public async ValueTask CreateService(CancellationToken cancellationToken) + => CreateServiceImpl( + await gitHubClientFactory.CreateClient(cancellationToken)); /// - public IAuthenticatedGitHubService CreateService(string accessToken) + public async ValueTask CreateService(string accessToken, CancellationToken cancellationToken) => CreateServiceImpl( - gitHubClientFactory.CreateClient( - accessToken ?? throw new ArgumentNullException(nameof(accessToken)))); + await gitHubClientFactory.CreateClient( + accessToken ?? throw new ArgumentNullException(nameof(accessToken)), + cancellationToken)); /// /// Create a . diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs index ec95fd16ab8..1808b4ca3c3 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs @@ -1,4 +1,7 @@ -using Octokit; +using System.Threading; +using System.Threading.Tasks; + +using Octokit; namespace Tgstation.Server.Host.Utils.GitHub { @@ -10,14 +13,25 @@ public interface IGitHubClientFactory /// /// Create a client. Low rate limit unless the server's GitHubAccessToken is set to bypass it. /// + /// The for the operation. /// A new . - IGitHubClient CreateClient(); + ValueTask CreateClient(CancellationToken cancellationToken); /// /// Create a client with authentication using a personal access token. /// /// The GitHub personal access token. + /// The for the operation. /// A new . - IGitHubClient CreateClient(string accessToken); + ValueTask CreateClient(string accessToken, CancellationToken cancellationToken); + + /// + /// Creates a GitHub App client for an installation. + /// + /// The private key . + /// The GitHub repository ID. + /// The for the operation. + /// A resulting in a new for the given or if authentication failed. + ValueTask CreateInstallationClient(string pem, long repositoryId, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs index f4e7be70f55..adb0ec88ebd 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs @@ -1,4 +1,7 @@ -namespace Tgstation.Server.Host.Utils.GitHub +using System.Threading; +using System.Threading.Tasks; + +namespace Tgstation.Server.Host.Utils.GitHub { /// /// Factory for s. @@ -8,14 +11,16 @@ public interface IGitHubServiceFactory /// /// Create a . /// - /// A new . - public IGitHubService CreateService(); + /// The for the operation. + /// A resulting in a new . + public ValueTask CreateService(CancellationToken cancellationToken); /// /// Create an . /// /// The access token to use for communication with GitHub. - /// A new . - public IAuthenticatedGitHubService CreateService(string accessToken); + /// The for the operation. + /// A resulting in a new . + public ValueTask CreateService(string accessToken, CancellationToken cancellationToken); } } diff --git a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs index 6f4fac0ae26..33058be75f4 100644 --- a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs @@ -1,5 +1,6 @@ using System; using System.Net.Http.Headers; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -57,14 +58,14 @@ public async Task TestCreateBasicClient() mockOptions.SetupGet(x => x.Value).Returns(gc); var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); - var client = factory.CreateClient(); + var client = await factory.CreateClient(CancellationToken.None); Assert.IsNotNull(client); var credentials = await client.Connection.CredentialStore.GetCredentials(); Assert.AreEqual(AuthenticationType.Anonymous, credentials.AuthenticationType); gc.GitHubAccessToken = "asdfasdfasdfasdfasdfasdf"; - client = factory.CreateClient(); + client = await factory.CreateClient(CancellationToken.None); Assert.IsNotNull(client); credentials = await client.Connection.CredentialStore.GetCredentials(); @@ -83,9 +84,9 @@ public async Task TestCreateTokenClient() mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); - Assert.ThrowsException(() => factory.CreateClient(null)); + await Assert.ThrowsExceptionAsync(() => factory.CreateClient(null, CancellationToken.None).AsTask()); - var client = factory.CreateClient("asdf"); + var client = await factory.CreateClient("asdf", CancellationToken.None); Assert.IsNotNull(client); var credentials = await client.Connection.CredentialStore.GetCredentials(); @@ -96,7 +97,7 @@ public async Task TestCreateTokenClient() } [TestMethod] - public void TestClientCaching() + public async Task TestClientCaching() { var mockApp = new Mock(); mockApp.SetupGet(x => x.ProductInfoHeaderValue).Returns(new ProductInfoHeaderValue("TGSTests", "1.2.3")).Verifiable(); @@ -105,10 +106,10 @@ public void TestClientCaching() mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); - var client1 = factory.CreateClient(); - var client2 = factory.CreateClient("asdf"); - var client3 = factory.CreateClient(); - var client4 = factory.CreateClient("asdf"); + var client1 = await factory.CreateClient(CancellationToken.None); + var client2 = await factory.CreateClient("asdf", CancellationToken.None); + var client3 = await factory.CreateClient(CancellationToken.None); + var client4 = await factory.CreateClient("asdf", CancellationToken.None); Assert.ReferenceEquals(client1, client3); Assert.ReferenceEquals(client2, client4); } diff --git a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs index daa9afdd4c2..24212eaa936 100644 --- a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs @@ -1,4 +1,6 @@ using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -28,27 +30,29 @@ public void TestConstructor() } [TestMethod] - public void TestCreateService() + public async Task TestCreateService() { var mockFactory = new Mock(); - mockFactory.Setup(x => x.CreateClient()).Returns(Mock.Of()).Verifiable(); +#pragma warning disable CA2012 // Use ValueTasks correctly + mockFactory.Setup(x => x.CreateClient(It.IsAny())).Returns(ValueTask.FromResult(Mock.Of())).Verifiable(); var mockToken = "asdf"; - mockFactory.Setup(x => x.CreateClient(mockToken)).Returns(Mock.Of()).Verifiable(); + mockFactory.Setup(x => x.CreateClient(mockToken, It.IsAny())).Returns(ValueTask.FromResult(Mock.Of())).Verifiable(); +#pragma warning restore CA2012 // Use ValueTasks correctly var mockOptions = new Mock>(); mockOptions.SetupGet(x => x.Value).Returns(new UpdatesConfiguration()); var factory = new GitHubServiceFactory(mockFactory.Object, Mock.Of(), mockOptions.Object); - Assert.ThrowsException(() => factory.CreateService(null)); + await Assert.ThrowsExceptionAsync(() => factory.CreateService(null, CancellationToken.None).AsTask()); Assert.AreEqual(0, mockFactory.Invocations.Count); - var result1 = factory.CreateService(); + var result1 = await factory.CreateService(CancellationToken.None); Assert.IsNotNull(result1); - var result2 = factory.CreateService(mockToken); + var result2 = factory.CreateService(mockToken, CancellationToken.None); Assert.IsNotNull(result2); mockFactory.VerifyAll(); diff --git a/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs b/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs index 2f657577820..d05faed9a65 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs @@ -1,4 +1,6 @@ using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -18,13 +20,13 @@ public DummyGitHubServiceFactory(ICryptographySuite cryptographySuite, ILogger CreateDummyService(); + public ValueTask CreateService(CancellationToken cancellationToken) => ValueTask.FromResult(CreateDummyService()); - public IAuthenticatedGitHubService CreateService(string accessToken) + public ValueTask CreateService(string accessToken, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(accessToken); - return CreateDummyService(); + return ValueTask.FromResult(CreateDummyService()); } TestingGitHubService CreateDummyService() => new TestingGitHubService(cryptographySuite, logger); diff --git a/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs b/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs index d104dbd6d3a..585dd569997 100644 --- a/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs +++ b/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs @@ -39,7 +39,7 @@ static TestingGitHubService() }); var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), Mock.Of>(), mockOptions.Object); - RealClient = gitHubClientFactory.CreateClient(); + RealClient = gitHubClientFactory.CreateClient(CancellationToken.None).GetAwaiter().GetResult(); } public static async Task InitializeAndInject(CancellationToken cancellationToken) From 12878f177fcada2897c6e5740ef7beb44b01987e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 13 Aug 2024 21:56:03 -0400 Subject: [PATCH 022/111] Adds version reporting telemetry --- .github/workflows/ci-pipeline.yml | 40 +++ README.md | 6 + .../Configuration/TelemetryConfiguration.cs | 33 +++ src/Tgstation.Server.Host/Core/Application.cs | 2 + .../Core/VersionReportingService.cs | 234 ++++++++++++++++++ .../TelemetryAppSerializedKeyAttribute.cs | 33 +++ .../Setup/SetupWizard.cs | 32 +++ .../Tgstation.Server.Host.csproj | 23 +- src/Tgstation.Server.Host/appsettings.yml | 4 + .../Setup/TestSetupWizard.cs | 9 +- .../Live/LiveTestingServer.cs | 1 + 11 files changed, 413 insertions(+), 4 deletions(-) create mode 100644 src/Tgstation.Server.Host/Configuration/TelemetryConfiguration.cs create mode 100644 src/Tgstation.Server.Host/Core/VersionReportingService.cs create mode 100644 src/Tgstation.Server.Host/Properties/TelemetryAppSerializedKeyAttribute.cs diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 54007f37f3e..375fb7dc40d 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -555,6 +555,8 @@ jobs: database-type: [ 'SqlServer', 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ] watchdog-type: [ 'Basic', 'Advanced' ] configuration: [ 'Debug', 'Release' ] + env: + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt runs-on: windows-latest steps: - name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later @@ -642,9 +644,18 @@ jobs: - name: Restore run: dotnet restore + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build run: dotnet build -c ${{ matrix.configuration }} tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -786,6 +797,8 @@ jobs: database-type: [ 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ] watchdog-type: [ 'Basic', 'Advanced' ] configuration: [ 'Debug', 'Release' ] + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt runs-on: ubuntu-latest steps: - name: Disable ptrace_scope @@ -848,9 +861,16 @@ jobs: - name: Restore run: dotnet restore + - name: Setup Telemetry Key File + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWindows tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj + - name: Delete Telemetry Key File + if: always() + run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -1194,6 +1214,8 @@ jobs: needs: start-ci-run-gate runs-on: ubuntu-latest if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt steps: - name: Install Native Dependencies run: | @@ -1244,6 +1266,9 @@ jobs: - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + - name: Setup Telemetry Key File + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Execute Build Script (Unsigned) if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev'))) run: sudo -E build/package/deb/build_package.sh @@ -1258,6 +1283,10 @@ jobs: gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1_amd64.changes gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1_amd64.buildinfo + - name: Delete Telemetry Key File + if: always() + run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Test Install run: | sudo mkdir /etc/tgstation-server @@ -1298,6 +1327,8 @@ jobs: needs: start-ci-run-gate runs-on: windows-latest if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt steps: - name: Install winget uses: Cyberboss/install-winget@v1 @@ -1331,9 +1362,18 @@ jobs: - name: Restore run: dotnet restore + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Host run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj diff --git a/README.md b/README.md index 794543dc854..c130e2d76bd 100644 --- a/README.md +++ b/README.md @@ -313,6 +313,12 @@ The following providers use the `ServerUrl` setting: - Keycloak - InvisionCommunity +- `Telemetry:DisableVersionReporting`: Prevents you installation and the version you're using from being reported on the source repository's deployments list + +- `Telemetry:ServerFriendlyName`: Prevents anonymous TGS version usage statistics from being sent to be displayed on the repository. + +- `Telemetry:VersionReportingRepositoryId`: The repository telemetry is sent to. For security reasons, this is not the main TGS repo. See the [tgstation-server-deployments](https://github.com/tgstation/tgstation-server-deployments) repository for more information. + ### Database Configuration If using a MariaDB/MySQL server, our client library [recommends you set 'utf8mb4' as your default charset](https://github.com/PomeloFoundation/Pomelo.EntityFrameworkCore.MySql#1-recommended-server-charset) disregard at your own risk. diff --git a/src/Tgstation.Server.Host/Configuration/TelemetryConfiguration.cs b/src/Tgstation.Server.Host/Configuration/TelemetryConfiguration.cs new file mode 100644 index 00000000000..7ec0361759d --- /dev/null +++ b/src/Tgstation.Server.Host/Configuration/TelemetryConfiguration.cs @@ -0,0 +1,33 @@ +namespace Tgstation.Server.Host.Configuration +{ + /// + /// Configuration options for telemetry. + /// + public sealed class TelemetryConfiguration + { + /// + /// The key for the the resides in. + /// + public const string Section = "Telemetry"; + + /// + /// The default value of . + /// + private const long DefaultVersionReportingRepositoryId = 841149827; // https://github.com/tgstation/tgstation-server-deployments + + /// + /// If version reporting telemetry is disabled. + /// + public bool DisableVersionReporting { get; set; } + + /// + /// The friendly name used on GitHub deployments for version reporting. If only the server will be shown. + /// + public string? ServerFriendlyName { get; set; } + + /// + /// The GitHub repository ID used for version reporting. + /// + public long? VersionReportingRepositoryId { get; set; } = DefaultVersionReportingRepositoryId; + } +} diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 36970000c59..7e981066c0d 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -143,6 +143,7 @@ public void ConfigureServices( services.UseStandardConfig(Configuration); services.UseStandardConfig(Configuration); services.UseStandardConfig(Configuration); + services.UseStandardConfig(Configuration); // enable options which give us config reloading services.AddOptions(); @@ -423,6 +424,7 @@ void AddTypedContext() services.AddSingleton(); services.AddSingleton(); services.AddHostedService(); + services.AddHostedService(); services.AddFileDownloader(); services.AddGitHub(); diff --git a/src/Tgstation.Server.Host/Core/VersionReportingService.cs b/src/Tgstation.Server.Host/Core/VersionReportingService.cs new file mode 100644 index 00000000000..7e03e12951d --- /dev/null +++ b/src/Tgstation.Server.Host/Core/VersionReportingService.cs @@ -0,0 +1,234 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using Octokit; + +using Tgstation.Server.Common.Extensions; +using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Properties; +using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; + +namespace Tgstation.Server.Host.Core +{ + /// + /// Handles TGS version reporting, if enabled. + /// + sealed class VersionReportingService : BackgroundService + { + /// + /// The for the . + /// + readonly IGitHubClientFactory gitHubClientFactory; + + /// + /// The for the . + /// + readonly IIOManager ioManager; + + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + + /// + /// The for the . + /// + readonly IAssemblyInformationProvider assemblyInformationProvider; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// The for the . + /// + readonly TelemetryConfiguration telemetryConfiguration; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + /// The value of . + /// The containing the value of . + /// The value of . + public VersionReportingService( + IGitHubClientFactory gitHubClientFactory, + IIOManager ioManager, + IAsyncDelayer asyncDelayer, + IAssemblyInformationProvider assemblyInformationProvider, + IOptions telemetryConfigurationOptions, + ILogger logger) + { + this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); + telemetryConfiguration = telemetryConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(telemetryConfigurationOptions)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (telemetryConfiguration.DisableVersionReporting) + { + logger.LogDebug("Version telemetry disabled"); + return; + } + + if (!telemetryConfiguration.VersionReportingRepositoryId.HasValue) + { + logger.LogError("Version reporting repository is misconfigured. Telemetry cannot be sent!"); + return; + } + + var attribute = TelemetryAppSerializedKeyAttribute.Instance; + if (attribute == null) + { + logger.LogDebug("TGS build configuration does not allow for version telemetry"); + return; + } + + logger.LogDebug("Starting..."); + + try + { + var telemetryIdFile = ioManager.ResolvePath( + ioManager.ConcatPath( + ioManager.GetPathInLocalDirectory(assemblyInformationProvider), + "telemetry.id")); + + Guid telemetryId; + if (!await ioManager.FileExists(telemetryIdFile, stoppingToken)) + { + telemetryId = Guid.NewGuid(); + await ioManager.WriteAllBytes(telemetryIdFile, Encoding.UTF8.GetBytes(telemetryId.ToString()), stoppingToken); + logger.LogInformation("Generated telemetry ID {telemetryId} and wrote to {file}", telemetryId, telemetryIdFile); + } + else + { + var contents = await ioManager.ReadAllBytes(telemetryIdFile, stoppingToken); + + string guidStr; + try + { + guidStr = Encoding.UTF8.GetString(contents); + } + catch (Exception ex) + { + logger.LogError(ex, "Cannot decode telemetry ID from installation file ({path}). Telemetry will not be sent!", telemetryIdFile); + return; + } + + if (!Guid.TryParse(guidStr, out telemetryId)) + { + logger.LogError("Cannot parse telemetry ID from installation file ({path}). Telemetry will not be sent!", telemetryIdFile); + return; + } + } + + var nextDelayHours = await TryReportVersion( + telemetryId, + attribute.SerializedKey, + telemetryConfiguration.VersionReportingRepositoryId.Value, + stoppingToken) + ? 24 + : 1; + + logger.LogDebug("Next version report in {hours} hours", nextDelayHours); + await asyncDelayer.Delay(TimeSpan.FromHours(nextDelayHours), stoppingToken); + } + catch (OperationCanceledException ex) + { + logger.LogTrace(ex, "Exiting..."); + } + catch (Exception ex) + { + logger.LogError(ex, "Crashed!"); + } + } + + /// + /// Make an attempt to report the current to the configured GitHub repository. + /// + /// The telemetry for the installation. + /// The serialized authentication for the . + /// The ID of the repository to send telemetry to. + /// The for the operation. + /// A resulting in if telemetry was reported successfully, otherwise. + async ValueTask TryReportVersion(Guid telemetryId, string serializedPem, long repositoryId, CancellationToken cancellationToken) + { + logger.LogDebug("Sending version telemetry..."); + + var serverFriendlyName = telemetryConfiguration.ServerFriendlyName; + if (String.IsNullOrWhiteSpace(serverFriendlyName)) + serverFriendlyName = null; + + logger.LogTrace( + "Repository ID: {repoId}, Server friendly name: {friendlyName}", + repositoryId, + serverFriendlyName == null + ? "(null)" + : $"\"{serverFriendlyName}\""); + try + { + var gitHubClient = await gitHubClientFactory.CreateInstallationClient( + serializedPem, + repositoryId, + cancellationToken); + + if (gitHubClient == null) + { + logger.LogWarning("Could not create GitHub client to connect to repository ID {repoId}!", repositoryId); + return false; + } + + // remove this lookup once https://github.com/octokit/octokit.net/pull/2960 is merged and released + var repository = await gitHubClient.Repository.Get(repositoryId); + + logger.LogTrace("Repository ID {id} resolved to {owner}/{name}", repositoryId, repository.Owner.Name, repository.Name); + + var inputs = new Dictionary + { + { "telemetry_id", telemetryId.ToString() }, + { "tgs_semver", assemblyInformationProvider.Version.Semver().ToString() }, + }; + + if (serverFriendlyName != null) + inputs.Add("server_friendly_name", serverFriendlyName); + + await gitHubClient.Actions.Workflows.CreateDispatch( + repository.Owner.Login, + repository.Name, + ".github/workflows/tgs_deployments_telemetry.yml", + new CreateWorkflowDispatch("main") + { + Inputs = inputs, + }); + + logger.LogTrace("Telemetry sent successfully"); + + return true; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to report version!"); + return false; + } + } + } +} diff --git a/src/Tgstation.Server.Host/Properties/TelemetryAppSerializedKeyAttribute.cs b/src/Tgstation.Server.Host/Properties/TelemetryAppSerializedKeyAttribute.cs new file mode 100644 index 00000000000..6d2ff1be5b8 --- /dev/null +++ b/src/Tgstation.Server.Host/Properties/TelemetryAppSerializedKeyAttribute.cs @@ -0,0 +1,33 @@ +using System; +using System.Reflection; + +namespace Tgstation.Server.Host.Properties +{ + /// + /// Attribute for bundling the GitHub App serialized private key used for version telemetry. + /// + [AttributeUsage(AttributeTargets.Assembly)] + sealed class TelemetryAppSerializedKeyAttribute : Attribute + { + /// + /// Return the 's instance of the . + /// + public static TelemetryAppSerializedKeyAttribute? Instance => Assembly + .GetExecutingAssembly() + .GetCustomAttribute(); + + /// + /// The serialized GitHub App Client ID and private key. + /// + public string SerializedKey { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public TelemetryAppSerializedKeyAttribute(string serializedKey) + { + SerializedKey = serializedKey ?? throw new ArgumentNullException(nameof(serializedKey)); + } + } +} diff --git a/src/Tgstation.Server.Host/Setup/SetupWizard.cs b/src/Tgstation.Server.Host/Setup/SetupWizard.cs index 04cc54a3bfd..387f2afe7a0 100644 --- a/src/Tgstation.Server.Host/Setup/SetupWizard.cs +++ b/src/Tgstation.Server.Host/Setup/SetupWizard.cs @@ -950,6 +950,31 @@ async ValueTask ParseAddress(string question) }; } + /// + /// Prompts the user to create a . + /// + /// The for the operation. + /// A resulting in the new . + async ValueTask ConfigureTelemetry(CancellationToken cancellationToken) + { + bool enableReporting = await PromptYesNo("Enable version telemetry? This anonymously reports the TGS version in use.", true, cancellationToken); + + string? serverFriendlyName = null; + if (enableReporting) + { + await console.WriteAsync("(Optional) Publically associate your reported version with a friendly name:", false, cancellationToken); + serverFriendlyName = await console.ReadLineAsync(false, cancellationToken); + if (String.IsNullOrWhiteSpace(serverFriendlyName)) + serverFriendlyName = null; + } + + return new TelemetryConfiguration + { + DisableVersionReporting = !enableReporting, + ServerFriendlyName = serverFriendlyName, + }; + } + /// /// Saves a given set to . /// @@ -961,6 +986,7 @@ async ValueTask ParseAddress(string question) /// The to save. /// The to save. /// The to save. + /// The to save. /// The for the operation. /// A representing the running operation. async ValueTask SaveConfiguration( @@ -972,6 +998,7 @@ async ValueTask SaveConfiguration( ElasticsearchConfiguration? elasticsearchConfiguration, ControlPanelConfiguration controlPanelConfiguration, SwarmConfiguration? swarmConfiguration, + TelemetryConfiguration? telemetryConfiguration, CancellationToken cancellationToken) { newGeneralConfiguration.ApiPort = hostingPort ?? GeneralConfiguration.DefaultApiPort; @@ -984,6 +1011,7 @@ async ValueTask SaveConfiguration( { ElasticsearchConfiguration.Section, elasticsearchConfiguration }, { ControlPanelConfiguration.Section, controlPanelConfiguration }, { SwarmConfiguration.Section, swarmConfiguration }, + { TelemetryConfiguration.Section, telemetryConfiguration }, }; var versionConverter = new VersionConverter(); @@ -1055,6 +1083,8 @@ async ValueTask RunWizard(string userConfigFileName, CancellationToken cancellat var swarmConfiguration = await ConfigureSwarm(cancellationToken); + var telemetryConfiguration = await ConfigureTelemetry(cancellationToken); + await console.WriteAsync(null, true, cancellationToken); await console.WriteAsync(String.Format(CultureInfo.InvariantCulture, "Configuration complete! Saving to {0}", userConfigFileName), true, cancellationToken); @@ -1067,6 +1097,7 @@ await SaveConfiguration( elasticSearchConfiguration, controlPanelConfiguration, swarmConfiguration, + telemetryConfiguration, cancellationToken); } @@ -1184,6 +1215,7 @@ await SaveConfiguration( AllowAnyOrigin = true, }, null, + null, cancellationToken); } else diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 94f3fcccb72..cdee07bb4a5 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -45,16 +45,33 @@ - + <_Parameter1>$(TgsConfigVersion) <_Parameter2>$(TgsInteropVersion) <_Parameter3>$(TgsWebpanelVersion) <_Parameter4>$(TgsHostWatchdogVersion) <_Parameter5>$(TgsMariaDBRedistVersion) - + + + + + + - + + + + + + + <_Parameter1>@(SerializedTelemetryKey) + + + diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 526f38ab386..68ab2bf729c 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -78,3 +78,7 @@ Swarm: # Should be left empty if using swarm mode is not desired # PublicAddress: # The public address of the swarm node # ControllerAddress: # Required on non-controller nodes. The internal address of the swarm controller's API'. Should be left empty on the controller itself # UpdateRequiredNodeCount: # The number of nodes expected to be in the swarm before initiating an update. This should count every server irrespective of whether or not they are the controller MINUS 1 +Telemetry: + DisableVersionReporting: false # Prevents you installation and the version you're using from being reported on the source repository's deployments list + ServerFriendlyName: null # Sets a friendly name for your server in reported telemetry. Must be unique. First come first serve + VersionReportingRepositoryId: 841149827 # GitHub repostiory ID where the tgs_version_telemetry workflow can be found diff --git a/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs b/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs index 823036a10b3..cd6e5c86a02 100644 --- a/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs +++ b/tests/Tgstation.Server.Host.Tests/Setup/TestSetupWizard.cs @@ -191,6 +191,8 @@ public async Task TestWithUserStupidity() "y", // swarm config "n", + // telemetry config + "n", //saved, now for second run //this time use defaults amap String.Empty, @@ -230,6 +232,9 @@ public async Task TestWithUserStupidity() "privatekey", "n", "http://controller.com", + // telemetry config + "y", + "telemetry name", //third run, we already hit all the code coverage so just get through it String.Empty, nameof(DatabaseType.MariaDB), @@ -267,7 +272,9 @@ public async Task TestWithUserStupidity() "https://controllerinternal.com", "https://controllerpublic.com", "privatekey", - "y" + "y", + // telemetry config + "n", }; var inputPos = 0; diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 645edac147d..f9ed0b926ab 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -155,6 +155,7 @@ public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth $"General:OpenDreamGitUrl={OpenDreamUrl}", $"Security:TokenExpiryMinutes=120", // timeouts are useless for us $"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}", + "Telemetry:DisableVersionReporting=true", }; swarmArgs = new List(); From e87259cfe5af1309cb31eebdcda55a843b3a5d88 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 13 Aug 2024 22:21:05 -0400 Subject: [PATCH 023/111] Bump webpanel to 6.1.0 --- build/Version.props | 1 - build/WebpanelVersion.props | 2 +- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 5 +++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/Version.props b/build/Version.props index 4f6a8945320..f524e2ee699 100644 --- a/build/Version.props +++ b/build/Version.props @@ -21,6 +21,5 @@ 11.4.2 - 1.22.21 diff --git a/build/WebpanelVersion.props b/build/WebpanelVersion.props index 806c5ed542d..2613f5510c0 100644 --- a/build/WebpanelVersion.props +++ b/build/WebpanelVersion.props @@ -1,6 +1,6 @@ - 5.9.0 + 6.1.0 diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index cdee07bb4a5..9e19034512c 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -29,13 +29,14 @@ - + + - + From 33aac9c4c936967002c363bd7a187e20f59cb7eb Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 13 Aug 2024 22:49:30 -0400 Subject: [PATCH 024/111] Report the shutdown and actually loop the telemetry reports --- .../Core/VersionReportingService.cs | 53 +++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/VersionReportingService.cs b/src/Tgstation.Server.Host/Core/VersionReportingService.cs index 7e03e12951d..dd0eef6f786 100644 --- a/src/Tgstation.Server.Host/Core/VersionReportingService.cs +++ b/src/Tgstation.Server.Host/Core/VersionReportingService.cs @@ -56,6 +56,11 @@ sealed class VersionReportingService : BackgroundService /// readonly TelemetryConfiguration telemetryConfiguration; + /// + /// The passed to . + /// + CancellationToken shutdownCancellationToken; + /// /// Initializes a new instance of the class. /// @@ -81,6 +86,13 @@ public VersionReportingService( this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } + /// + public override Task StopAsync(CancellationToken cancellationToken) + { + shutdownCancellationToken = cancellationToken; + return base.StopAsync(cancellationToken); + } + /// protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -141,20 +153,41 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } - var nextDelayHours = await TryReportVersion( + try + { + while (!stoppingToken.IsCancellationRequested) + { + var nextDelayHours = await TryReportVersion( + telemetryId, + attribute.SerializedKey, + telemetryConfiguration.VersionReportingRepositoryId.Value, + false, + stoppingToken) + ? 24 + : 1; + + logger.LogDebug("Next version report in {hours} hours", nextDelayHours); + await asyncDelayer.Delay(TimeSpan.FromHours(nextDelayHours), stoppingToken); + } + } + catch (OperationCanceledException ex) + { + logger.LogTrace(ex, "Inner cancellation"); + } + + shutdownCancellationToken.ThrowIfCancellationRequested(); + + logger.LogDebug("Sending shutdown telemetry"); + await TryReportVersion( telemetryId, attribute.SerializedKey, telemetryConfiguration.VersionReportingRepositoryId.Value, - stoppingToken) - ? 24 - : 1; - - logger.LogDebug("Next version report in {hours} hours", nextDelayHours); - await asyncDelayer.Delay(TimeSpan.FromHours(nextDelayHours), stoppingToken); + true, + shutdownCancellationToken); } catch (OperationCanceledException ex) { - logger.LogTrace(ex, "Exiting..."); + logger.LogTrace(ex, "Exiting due to outer cancellation..."); } catch (Exception ex) { @@ -168,9 +201,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) /// The telemetry for the installation. /// The serialized authentication for the . /// The ID of the repository to send telemetry to. + /// If this is shutdown telemetry. /// The for the operation. /// A resulting in if telemetry was reported successfully, otherwise. - async ValueTask TryReportVersion(Guid telemetryId, string serializedPem, long repositoryId, CancellationToken cancellationToken) + async ValueTask TryReportVersion(Guid telemetryId, string serializedPem, long repositoryId, bool shutdown, CancellationToken cancellationToken) { logger.LogDebug("Sending version telemetry..."); @@ -206,6 +240,7 @@ async ValueTask TryReportVersion(Guid telemetryId, string serializedPem, l { { "telemetry_id", telemetryId.ToString() }, { "tgs_semver", assemblyInformationProvider.Version.Semver().ToString() }, + { "shutdown", shutdown ? "true" : "false" }, }; if (serverFriendlyName != null) From 2c4a3d79678ac3220337932b329f577e0b76147a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 13 Aug 2024 23:12:55 -0400 Subject: [PATCH 025/111] Enable corepack on command line. Fix missed telemetry build in Windows .exe deployment --- .github/CONTRIBUTING.md | 2 + .github/workflows/ci-pipeline.yml | 38 ++++++++++++++----- .../Tgstation.Server.Host.csproj | 5 +-- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 233c7fb51e5..1c6c65e278c 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -36,6 +36,8 @@ You can of course, as always, ask for help at [#coderbus](irc://irc.rizon.net/co You need the .NET 8.0 SDK, node>=v20, and npm>=v5.7 (in your PATH) to compile the server. On Linux, you also need the `libgdiplus` package installed to generate icons. +You need to run `corepack enable` to configure node to correctly build the webpanel. + The recommended IDE is Visual Studio 2022 or VSCode. In order to build the service version and/or the Windows installer you need a to run on Windows. diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 375fb7dc40d..a3ec6a8c186 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -475,6 +475,9 @@ jobs: - name: Restore run: dotnet restore + - name: Enable Corepack + run: corepack enable + - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWindows @@ -526,6 +529,9 @@ jobs: - name: Restore run: dotnet restore + - name: Enable Corepack + run: corepack enable + - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWix @@ -644,6 +650,9 @@ jobs: - name: Restore run: dotnet restore + - name: Enable Corepack + run: corepack enable + - name: Setup Telemetry Key File shell: bash run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} @@ -861,6 +870,9 @@ jobs: - name: Restore run: dotnet restore + - name: Enable Corepack + run: corepack enable + - name: Setup Telemetry Key File run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} @@ -1327,8 +1339,6 @@ jobs: needs: start-ci-run-gate runs-on: windows-latest if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') - env: - TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt steps: - name: Install winget uses: Cyberboss/install-winget@v1 @@ -1362,18 +1372,12 @@ jobs: - name: Restore run: dotnet restore - - name: Setup Telemetry Key File - shell: bash - run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Enable Corepack + run: corepack enable - name: Build Host run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj - - name: Delete Telemetry Key File - shell: bash - if: always() - run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} - - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj @@ -1769,6 +1773,8 @@ jobs: needs: [deploy-dm, deploy-http, deployment-gate] runs-on: windows-latest if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && github.event.ref == 'refs/heads/master' && contains(github.event.head_commit.message, '[TGSDeploy]')) + env: + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1789,9 +1795,21 @@ jobs: # We need to rebuild the installer.exe so it can be properly signed + - name: Enable Corepack + run: corepack enable + + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Host run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 9e19034512c..999dff04a84 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -29,14 +29,13 @@ - - + - + From dca0946cb67adf18f485c44f6211795ac9649ba5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:38:01 +0000 Subject: [PATCH 026/111] Bump Microsoft.AspNetCore.SignalR.Client from 8.0.7 to 8.0.8 Bumps [Microsoft.AspNetCore.SignalR.Client](https://github.com/dotnet/aspnetcore) from 8.0.7 to 8.0.8. - [Release notes](https://github.com/dotnet/aspnetcore/releases) - [Changelog](https://github.com/dotnet/aspnetcore/blob/main/docs/ReleasePlanning.md) - [Commits](https://github.com/dotnet/aspnetcore/compare/v8.0.7...v8.0.8) --- updated-dependencies: - dependency-name: Microsoft.AspNetCore.SignalR.Client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Client/Tgstation.Server.Client.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 89c048c68f2..fdb60b80d83 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -11,7 +11,7 @@ - + From 3e8cbd89158c6cfcb8ab809fdfcf127e87a7c89b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:38:27 +0000 Subject: [PATCH 027/111] Bump Microsoft.AspNetCore.Authentication.JwtBearer from 8.0.7 to 8.0.8 Bumps [Microsoft.AspNetCore.Authentication.JwtBearer](https://github.com/dotnet/aspnetcore) from 8.0.7 to 8.0.8. - [Release notes](https://github.com/dotnet/aspnetcore/releases) - [Changelog](https://github.com/dotnet/aspnetcore/blob/main/docs/ReleasePlanning.md) - [Commits](https://github.com/dotnet/aspnetcore/compare/v8.0.7...v8.0.8) --- updated-dependencies: - dependency-name: Microsoft.AspNetCore.Authentication.JwtBearer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj index 01518a39b01..c092a103f7f 100644 --- a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj +++ b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj @@ -10,7 +10,7 @@ - + From 745cf554275fc12870dbfab82099bd83c9cf4d6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:41:41 +0000 Subject: [PATCH 028/111] Bump Microsoft.EntityFrameworkCore from 8.0.7 to 8.0.8 Bumps [Microsoft.EntityFrameworkCore](https://github.com/dotnet/efcore) from 8.0.7 to 8.0.8. - [Release notes](https://github.com/dotnet/efcore/releases) - [Commits](https://github.com/dotnet/efcore/compare/v8.0.7...v8.0.8) --- updated-dependencies: - dependency-name: Microsoft.EntityFrameworkCore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 94f3fcccb72..5c5e611b2c5 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -84,7 +84,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive From ce4b22dd32e6e6d1aec6f2b39a25e728ed7cd6b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:42:59 +0000 Subject: [PATCH 029/111] Bump MSTest.TestAdapter from 3.5.1 to 3.5.2 Bumps [MSTest.TestAdapter](https://github.com/microsoft/testfx) from 3.5.1 to 3.5.2. - [Release notes](https://github.com/microsoft/testfx/releases) - [Changelog](https://github.com/microsoft/testfx/blob/main/docs/Changelog.md) - [Commits](https://github.com/microsoft/testfx/compare/v3.5.1...v3.5.2) --- updated-dependencies: - dependency-name: MSTest.TestAdapter dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build/TestCommon.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/TestCommon.props b/build/TestCommon.props index 98703bdbfd6..3f787bc5729 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -18,7 +18,7 @@ - + From 451708cad2fec753ff0141b33d674ba3275c5dc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:43:35 +0000 Subject: [PATCH 030/111] Bump Microsoft.AspNetCore.Mvc.NewtonsoftJson from 8.0.7 to 8.0.8 Bumps [Microsoft.AspNetCore.Mvc.NewtonsoftJson](https://github.com/dotnet/aspnetcore) from 8.0.7 to 8.0.8. - [Release notes](https://github.com/dotnet/aspnetcore/releases) - [Changelog](https://github.com/dotnet/aspnetcore/blob/main/docs/ReleasePlanning.md) - [Commits](https://github.com/dotnet/aspnetcore/compare/v8.0.7...v8.0.8) --- updated-dependencies: - dependency-name: Microsoft.AspNetCore.Mvc.NewtonsoftJson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 94f3fcccb72..0d9cbf10bca 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -78,7 +78,7 @@ - + From 2c383fddcb5b72a2fc2fc1b386dcc85e3f369c51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:44:48 +0000 Subject: [PATCH 031/111] Bump MSTest.TestFramework from 3.5.1 to 3.5.2 Bumps [MSTest.TestFramework](https://github.com/microsoft/testfx) from 3.5.1 to 3.5.2. - [Release notes](https://github.com/microsoft/testfx/releases) - [Changelog](https://github.com/microsoft/testfx/blob/main/docs/Changelog.md) - [Commits](https://github.com/microsoft/testfx/compare/v3.5.1...v3.5.2) --- updated-dependencies: - dependency-name: MSTest.TestFramework dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build/TestCommon.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/TestCommon.props b/build/TestCommon.props index 98703bdbfd6..d77dad1b08b 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -20,7 +20,7 @@ - + From 2ec3d836d0c54543d3b71824e7809594bcbee9bd Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 14 Aug 2024 07:04:33 -0400 Subject: [PATCH 032/111] Maybe corepack will work if we setup node? --- .github/workflows/ci-pipeline.yml | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index a3ec6a8c186..abd39d54560 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -40,6 +40,7 @@ env: OD_MIN_COMPAT_DOTNET_VERSION: 7 OD_DOTNET_VERSION: 8 TGS_DOTNET_QUALITY: ga + TGS_WEBPANEL_NODE_VERSION: 20.x TGS_TEST_GITHUB_TOKEN: ${{ secrets.LIVE_TESTS_TOKEN }} TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} PACKAGING_PRIVATE_KEY_PASSPHRASE: ${{ secrets.PACKAGING_PRIVATE_KEY_PASSPHRASE }} @@ -462,6 +463,16 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -516,6 +527,11 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -581,6 +597,11 @@ jobs: ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Set TGS_TEST_DUMP_API_SPEC if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'SqlServer' }} run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV @@ -828,6 +849,11 @@ jobs: ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Set Sqlite Connection Info if: ${{ matrix.database-type == 'Sqlite' }} run: | @@ -1351,6 +1377,11 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -1782,6 +1813,11 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Checkout uses: actions/checkout@v4 From 6d41ffc17ba7fdb5502403861c2ded17b8e9fee1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 14 Aug 2024 07:10:12 -0400 Subject: [PATCH 033/111] Fail CI build if telemetry key isn't set --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 999dff04a84..48025ddd9b0 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -59,7 +59,7 @@ - + + + + + From 8f348595fbce1b50762affbaf8d8716d6214828e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 14 Aug 2024 22:45:13 -0400 Subject: [PATCH 034/111] Fix docker build --- .github/workflows/ci-pipeline.yml | 12 +++++++++++- build/Dockerfile | 10 ++++++++-- .../Tgstation.Server.Host.csproj | 1 + 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index abd39d54560..867323f977a 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -424,6 +424,8 @@ jobs: runs-on: ubuntu-latest needs: start-ci-run-gate if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: tgs_telemetry_key.txt steps: - name: Checkout (Branch) uses: actions/checkout@v4 @@ -435,8 +437,16 @@ jobs: with: ref: "refs/pull/${{ github.event.number }}/merge" + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Docker Image - run: docker build . -f build/Dockerfile + run: docker build . -f build/Dockerfile --build-arg TGS_TELEMETRY_KEY_FILE=${{ env.TGS_TELEMETRY_KEY_FILE }} + + - name: Delete Telemetry Key File + if: always() + run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} linux-unit-tests: name: Linux Tests diff --git a/build/Dockerfile b/build/Dockerfile index cba330156e2..6309cec6187 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,5 +1,8 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim AS build +# Set in CI +ARG TGS_TELEMETRY_KEY_FILE= + # install node and npm # replace shell with bash so we can source files RUN curl --silent -o- https://raw.githubusercontent.com/creationix/nvm/v0.39.1/install.sh | sh @@ -17,7 +20,8 @@ RUN . $NVM_DIR/nvm.sh \ && apt-get install -y \ dos2unix \ libgdiplus \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && corepack enable # Build web control panel WORKDIR /repo/build @@ -45,7 +49,9 @@ RUN dotnet publish -c Release -o /app \ && build/RemoveUnsupportedRuntimes.sh /app WORKDIR /repo/src/Tgstation.Server.Host -RUN dotnet publish -c Release -o /app/lib/Default \ + +RUN export TGS_TELEMETRY_KEY_FILE="${TGS_TELEMETRY_KEY_FILE}" \ + && dotnet publish -c Release -o /app/lib/Default \ && cd ../.. \ && build/RemoveUnsupportedRuntimes.sh /app/lib/Default \ && mv /app/lib/Default/appsettings* /app diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 48025ddd9b0..0a3a5a3a06f 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -60,6 +60,7 @@ + Date: Wed, 14 Aug 2024 22:57:35 -0400 Subject: [PATCH 035/111] Hopefully fix the remainder of CI --- .github/workflows/ci-pipeline.yml | 53 +++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 867323f977a..70c57bbd2ff 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -109,6 +109,8 @@ jobs: permissions: security-events: write actions: read + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Setup dotnet @@ -132,9 +134,16 @@ jobs: with: languages: csharp + - name: Setup Telemetry Key File + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true + - name: Delete Telemetry Key File + if: always() + run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 with: @@ -459,6 +468,7 @@ jobs: env: TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }} + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt runs-on: ubuntu-latest steps: - name: Install x86 libc Dependencies @@ -473,11 +483,6 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Setup - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - name: Setup Node.JS uses: actions/setup-node@v4 with: @@ -499,9 +504,16 @@ jobs: - name: Enable Corepack run: corepack enable + - name: Setup Telemetry Key File + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWindows + - name: Delete Telemetry Key File + if: always() + run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -529,6 +541,7 @@ jobs: env: TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }} + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt runs-on: windows-latest steps: - name: Setup dotnet @@ -558,9 +571,18 @@ jobs: - name: Enable Corepack run: corepack enable + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build run: dotnet build -c ${{ matrix.configuration }}NoWix + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond @@ -1290,6 +1312,11 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Setup Node.JS + uses: actions/setup-node@v4 + with: + node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} + - name: Override /usr/bin/dotnet run: | DOTNET_PATH=$(which dotnet) @@ -1314,6 +1341,9 @@ jobs: - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + - name: Enable Corepack + run: corepack enable + - name: Setup Telemetry Key File run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} @@ -1375,6 +1405,8 @@ jobs: needs: start-ci-run-gate runs-on: windows-latest if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') + env: + TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt steps: - name: Install winget uses: Cyberboss/install-winget@v1 @@ -1416,9 +1448,18 @@ jobs: - name: Enable Corepack run: corepack enable + - name: Setup Telemetry Key File + shell: bash + run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Host run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj + - name: Delete Telemetry Key File + shell: bash + if: always() + run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj @@ -1839,7 +1880,7 @@ jobs: cd build/package/winget dotnet tool restore -# We need to rebuild the installer.exe so it can be properly signed + # We need to rebuild the installer.exe so it can be properly signed - name: Enable Corepack run: corepack enable From 909fca38194415e1a15a4ed1ac7df4eed5bffc6c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 14 Aug 2024 23:00:36 -0400 Subject: [PATCH 036/111] Bump dotnet-ef tool from 8.0.7 to 8.0.8 --- src/Tgstation.Server.Host/.config/dotnet-tools.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/.config/dotnet-tools.json b/src/Tgstation.Server.Host/.config/dotnet-tools.json index af52642792d..a3847dcdfb2 100644 --- a/src/Tgstation.Server.Host/.config/dotnet-tools.json +++ b/src/Tgstation.Server.Host/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-ef": { - "version": "8.0.7", + "version": "8.0.8", "commands": [ "dotnet-ef" ] From bf4d415aa242468621fc0fa9c12e6253a4f90a37 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 15 Aug 2024 06:43:54 -0400 Subject: [PATCH 037/111] Fix telemetry key file path in Dockerfile --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 6309cec6187..b301591833e 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -50,7 +50,7 @@ RUN dotnet publish -c Release -o /app \ WORKDIR /repo/src/Tgstation.Server.Host -RUN export TGS_TELEMETRY_KEY_FILE="${TGS_TELEMETRY_KEY_FILE}" \ +RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \ && dotnet publish -c Release -o /app/lib/Default \ && cd ../.. \ && build/RemoveUnsupportedRuntimes.sh /app/lib/Default \ From d034f71cbbe86aeb57245f19d529cc35cda29e8a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 15 Aug 2024 07:04:27 -0400 Subject: [PATCH 038/111] Hopefully fix corepack issue in debian build --- .github/workflows/ci-pipeline.yml | 16 +--------------- build/package/deb/build_package.sh | 3 ++- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 70c57bbd2ff..62c07eddc85 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1291,7 +1291,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386 gnupg2 xmlstarlet libgdiplus + sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386 gnupg2 - name: Import GPG Key if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) @@ -1306,17 +1306,6 @@ jobs: sudo apt-get update sudo apt-get install -y dotnet-sdk-${{ env.TGS_DOTNET_VERSION }}.0 - - name: Setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Setup Node.JS - uses: actions/setup-node@v4 - with: - node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }} - - name: Override /usr/bin/dotnet run: | DOTNET_PATH=$(which dotnet) @@ -1341,9 +1330,6 @@ jobs: - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml - - name: Enable Corepack - run: corepack enable - - name: Setup Telemetry Key File run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} diff --git a/build/package/deb/build_package.sh b/build/package/deb/build_package.sh index 1a25546da88..f4c3dec3ca7 100755 --- a/build/package/deb/build_package.sh +++ b/build/package/deb/build_package.sh @@ -35,6 +35,8 @@ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.co apt-get update apt-get install nodejs dotnet-sdk-8.0 -y +corepack enable + CURRENT_COMMIT=$(git rev-parse HEAD) rm -rf packaging @@ -67,7 +69,6 @@ cp build/tgstation-server.service debian/ SIGN_COMMAND="$SCRIPT_DIR/wrap_gpg.sh" rm -f /tmp/tgs_wrap_gpg_output.log - set +e if [[ -z "$PACKAGING_KEYGRIP" ]]; then From d4a13354a1d00ee593370a336168245c76e74896 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 15 Aug 2024 07:06:27 -0400 Subject: [PATCH 039/111] Properly switch to using gnupg2 for .deb --- .github/workflows/ci-pipeline.yml | 2 +- build/package/deb/build_package.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 62c07eddc85..f8414c188b2 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1291,7 +1291,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386 gnupg2 + sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386 - name: Import GPG Key if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) diff --git a/build/package/deb/build_package.sh b/build/package/deb/build_package.sh index f4c3dec3ca7..a48071f2193 100755 --- a/build/package/deb/build_package.sh +++ b/build/package/deb/build_package.sh @@ -18,7 +18,7 @@ apt-get install -y \ devscripts \ ca-certificates \ curl \ - gnupg \ + gnupg2 \ xmlstarlet \ libgdiplus From daa00e775d1ad413cd2f44434fa14df9258b1635 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 11:49:25 +0000 Subject: [PATCH 040/111] Bump Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson Bumps [Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson](https://github.com/dotnet/aspnetcore) from 8.0.7 to 8.0.8. - [Release notes](https://github.com/dotnet/aspnetcore/releases) - [Changelog](https://github.com/dotnet/aspnetcore/blob/main/docs/ReleasePlanning.md) - [Commits](https://github.com/dotnet/aspnetcore/compare/v8.0.7...v8.0.8) --- updated-dependencies: - dependency-name: Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Client/Tgstation.Server.Client.csproj | 2 +- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index fdb60b80d83..3bca3608474 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 0d9cbf10bca..ae09482924a 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -80,7 +80,7 @@ - + From 8f62ceae81899a3af045b4233d2570f068def173 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:57:03 +0000 Subject: [PATCH 041/111] Bump Microsoft.EntityFrameworkCore.Design from 8.0.7 to 8.0.8 Bumps [Microsoft.EntityFrameworkCore.Design](https://github.com/dotnet/efcore) from 8.0.7 to 8.0.8. - [Release notes](https://github.com/dotnet/efcore/releases) - [Commits](https://github.com/dotnet/efcore/compare/v8.0.7...v8.0.8) --- updated-dependencies: - dependency-name: Microsoft.EntityFrameworkCore.Design dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index d0cde05faee..7315cd7548d 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -86,7 +86,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive From 036b76bd6042bc9a3c6084835071e50b7d578f8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 23:00:57 +0000 Subject: [PATCH 042/111] Bump Microsoft.EntityFrameworkCore.SqlServer from 8.0.7 to 8.0.8 Bumps [Microsoft.EntityFrameworkCore.SqlServer](https://github.com/dotnet/efcore) from 8.0.7 to 8.0.8. - [Release notes](https://github.com/dotnet/efcore/releases) - [Commits](https://github.com/dotnet/efcore/compare/v8.0.7...v8.0.8) --- updated-dependencies: - dependency-name: Microsoft.EntityFrameworkCore.SqlServer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index d0cde05faee..a5213994a98 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -92,7 +92,7 @@ - + From 052ea768e34e93d8824775198369204b8c12113a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:33:33 +0000 Subject: [PATCH 043/111] Bump Microsoft.EntityFrameworkCore.InMemory from 8.0.7 to 8.0.8 Bumps [Microsoft.EntityFrameworkCore.InMemory](https://github.com/dotnet/efcore) from 8.0.7 to 8.0.8. - [Release notes](https://github.com/dotnet/efcore/releases) - [Commits](https://github.com/dotnet/efcore/compare/v8.0.7...v8.0.8) --- updated-dependencies: - dependency-name: Microsoft.EntityFrameworkCore.InMemory dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .../Tgstation.Server.Host.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj index 5e2a7f71e96..261c136fc7b 100644 --- a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj +++ b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj @@ -7,7 +7,7 @@ - + From 68a4f5940fda469ed8d6a6733ff2e7ca4f8ee7ac Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 10:03:48 -0400 Subject: [PATCH 044/111] Readd necessary setup-dotnet to .deb build --- .github/workflows/ci-pipeline.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index f8414c188b2..cd13a5c5326 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1306,6 +1306,12 @@ jobs: sudo apt-get update sudo apt-get install -y dotnet-sdk-${{ env.TGS_DOTNET_VERSION }}.0 + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Override /usr/bin/dotnet run: | DOTNET_PATH=$(which dotnet) From 8470e682b47e372964a6970eb5437c91d138a0cf Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 12:55:12 -0400 Subject: [PATCH 045/111] Redo CI Pipeline security to prevent duplicate checks --- .github/workflows/ci-pipeline.yml | 167 ++++-------------- .github/workflows/ci-security.yml | 70 ++++++++ .../Tgstation.Server.ReleaseNotes/Program.cs | 28 --- 3 files changed, 100 insertions(+), 165 deletions(-) create mode 100644 .github/workflows/ci-security.yml diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 54007f37f3e..b3c1a9f0847 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -29,11 +29,11 @@ on: branches: - dev - master - pull_request_target: - types: [ opened, reopened, labeled, synchronize ] - branches: - - dev - - master + workflow_dispatch: + inputs: + pull_request_number: + description: 'Pull Request Number' + required: true env: TGS_DOTNET_VERSION: 8 @@ -46,69 +46,16 @@ env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} concurrency: - group: "ci-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" + group: "ci-${{ github.head_ref || (github.event_name != 'push' && github.event_name != 'schedule' && github.event.inputs.pull_request_number) || github.run_id }}-${{ github.event_name }}" cancel-in-progress: true jobs: - security-checkpoint: - name: Check CI Clearance - runs-on: ubuntu-latest - permissions: - pull-requests: write - if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' - steps: - - name: Comment on new Fork PR - if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 - uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 - with: - message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Please note that any changes to the workflow file will not be reflected in the run. - - - name: Comment on dependabot PR - if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id == 49699333 - uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 - with: - message: Set the milestone to the next minor version, check for supply chain attacks, and then add the `CI Cleared` label to allow CI to run. - - - name: "Remove Stale 'CI Cleared' Label" - if: github.event.action == 'synchronize' || github.event.action == 'reopened' - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 - with: - labels: CI Cleared - - - name: "Remove 'CI Approval Required' Label" - if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && contains(github.event.pull_request.labels.*.name, 'CI Cleared')) - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 - with: - labels: CI Approval Required - - - name: "Add 'CI Approval Required' Label" - if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) - uses: actions-ecosystem/action-add-labels@bd52874380e3909a1ac983768df6976535ece7f8 - with: - labels: CI Approval Required - github_token: ${{ github.token }} - - - name: Fail Clearance Check if PR has Unlabeled new Commits from User - if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) - run: exit 1 - - start-ci-run-gate: - name: CI Start Gate - needs: security-checkpoint - runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && (needs.security-checkpoint.result == 'success' || (needs.security-checkpoint.result == 'skipped' && (github.event_name == 'push' || github.event_name == 'schedule' || ((github.event.pull_request.head.repo.id == github.event.pull_request.base.repo.id && github.event.pull_request.user.id != 49699333) && github.event_name != 'pull_request_target'))))) - steps: - - name: GitHub Requires at Least One Step for a Job - run: exit 0 - code-scanning: name: Code Scanning - needs: start-ci-run-gate runs-on: ubuntu-latest permissions: security-events: write actions: read - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -124,7 +71,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Initialize CodeQL uses: github/codeql-action/init@v3 @@ -141,8 +88,6 @@ jobs: dmapi-build: name: Build DMAPI - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -200,7 +145,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Build DMAPI Test Project run: | @@ -224,8 +169,6 @@ jobs: opendream-build: name: Build DMAPI (OpenDream) - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -254,7 +197,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Checkout OpenDream run: | @@ -287,8 +230,6 @@ jobs: efcore-version-match: name: Check Nuget Versions Match Tools runs-on: ubuntu-latest - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Checkout (Branch) uses: actions/checkout@v4 @@ -298,7 +239,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Retrieve dotnet-ef Tool Version id: dotnet-ef-tool @@ -345,8 +286,6 @@ jobs: pages-build: name: Build gh-pages runs-on: ubuntu-latest - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -362,7 +301,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -421,8 +360,6 @@ jobs: docker-build: name: Build Docker Image runs-on: ubuntu-latest - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Checkout (Branch) uses: actions/checkout@v4 @@ -432,15 +369,13 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Build Docker Image run: docker build . -f build/Dockerfile linux-unit-tests: name: Linux Tests - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -470,7 +405,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -496,8 +431,6 @@ jobs: windows-unit-tests: name: Windows Tests - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') strategy: fail-fast: false matrix: @@ -521,7 +454,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -548,7 +481,6 @@ jobs: windows-integration-test: name: Windows Live Tests needs: [dmapi-build, opendream-build] - if: (!(cancelled() || failure()) && needs.dmapi-build.result == 'success' && needs.opendream-build.result == 'success') strategy: fail-fast: false matrix: @@ -637,7 +569,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -738,7 +670,6 @@ jobs: linux-integration-tests: name: Linux Live Tests needs: [dmapi-build, opendream-build] - if: (!(cancelled() || failure()) && needs.dmapi-build.result == 'success' && needs.opendream-build.result == 'success') services: # We start all dbs here so we can just code the stuff once mssql: image: ${{ (matrix.database-type == 'SqlServer') && 'mcr.microsoft.com/mssql/server:2019-latest' || '' }} @@ -843,7 +774,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -908,7 +839,6 @@ jobs: validate-openapi-spec: name: OpenAPI Spec Validation needs: windows-integration-test - if: (!(cancelled() || failure()) && needs.windows-integration-test.result == 'success') runs-on: ubuntu-latest steps: - name: Install IBM OpenAPI Validator @@ -922,7 +852,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Retrieve OpenAPI Spec uses: actions/download-artifact@v4 @@ -936,7 +866,6 @@ jobs: upload-code-coverage: name: Upload Code Coverage needs: [linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-test] - if: (!(cancelled() || failure()) && needs.linux-unit-tests.result == 'success' && needs.linux-integration-tests.result == 'success' && needs.windows-unit-tests.result == 'success' && needs.windows-integration-test.result == 'success') runs-on: ubuntu-latest steps: - name: Checkout (Branch) @@ -947,7 +876,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Retrieve Linux Unit Test Coverage (Debug) uses: actions/download-artifact@v4 @@ -1191,9 +1120,7 @@ jobs: build-deb: name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595 - needs: start-ci-run-gate runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Install Native Dependencies run: | @@ -1235,7 +1162,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Parse TGS version run: | @@ -1295,9 +1222,7 @@ jobs: build-msi: name: Build Windows Installer .exe - needs: start-ci-run-gate runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Install winget uses: Cyberboss/install-winget@v1 @@ -1318,7 +1243,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Restore Wix dotnet Tool run: | @@ -1426,8 +1351,6 @@ jobs: check-winget-pr-template: name: Check winget-pkgs Pull Request Template is up to date - needs: start-ci-run-gate - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -1450,7 +1373,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -1465,43 +1388,15 @@ jobs: name: CI Completion Gate needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match ] runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.code-scanning.result == 'success') steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Checkout (Branch) - uses: actions/checkout@v4 - if: github.event_name == 'push' || github.event_name == 'schedule' - - - name: Checkout (PR Merge) - uses: actions/checkout@v4 - if: github.event_name != 'push' && github.event_name != 'schedule' - with: - ref: "refs/pull/${{ github.event.number }}/merge" - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - - name: Run ReleaseNotes Create CI Completion Check (PR HEAD) - if: github.event_name != 'push' && github.event_name != 'schedule' - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ci-completion-check ${{ github.event.pull_request.head.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} - - - name: Run ReleaseNotes Create CI Completion Check (Branch) - if: github.event_name == 'push' || github.event_name == 'schedule' - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ci-completion-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + - name: GitHub Requires at Least One Step for a Job + run: exit 0 deployment-gate: name: Deployment Start Gate needs: ci-completion-gate runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.ci-completion-gate.result == 'success' && github.event_name == 'push' && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) + if: github.event_name == 'push' && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev') steps: - name: GitHub Requires at Least One Step for a Job run: exit 0 @@ -1510,7 +1405,7 @@ jobs: name: Deploy HTTP API needs: deployment-gate runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[APIDeploy]')) + if: contains(github.event.head_commit.message, '[APIDeploy]') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1582,7 +1477,7 @@ jobs: name: Deploy DreamMaker API needs: deployment-gate runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[DMDeploy]')) + if: contains(github.event.head_commit.message, '[DMDeploy]') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1653,7 +1548,7 @@ jobs: name: Deploy Nuget Packages needs: deployment-gate runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[NugetDeploy]')) + if: contains(github.event.head_commit.message, '[NugetDeploy]') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1704,7 +1599,7 @@ jobs: name: Ensure TGS Release is Latest GitHub Release needs: [deploy-dm, deploy-http] runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && (needs.deploy-dm.result == 'success' || needs.deploy-http.result == 'success') && !contains(github.event.head_commit.message, '[TGSDeploy]')) + if: (!contains(github.event.head_commit.message, '[TGSDeploy]')) steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1728,7 +1623,7 @@ jobs: name: Deploy TGS needs: [deploy-dm, deploy-http, deployment-gate] runs-on: windows-latest - if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && github.event.ref == 'refs/heads/master' && contains(github.event.head_commit.message, '[TGSDeploy]')) + if: github.event.ref == 'refs/heads/master' && contains(github.event.head_commit.message, '[TGSDeploy]') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 @@ -1747,9 +1642,7 @@ jobs: cd build/package/winget dotnet tool restore -# We need to rebuild the installer.exe so it can be properly signed - - - name: Build Host + - name: Build Host # We need to rebuild the installer.exe so it can be properly signed run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj - name: Build Service diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml new file mode 100644 index 00000000000..3a860fb5a9f --- /dev/null +++ b/.github/workflows/ci-security.yml @@ -0,0 +1,70 @@ +name: 'CI Security' + +on: + pull_request_target: + types: [ opened, reopened, labeled, synchronize ] + branches: + - dev + - master + +concurrency: + group: "ci-security-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" + cancel-in-progress: true + +jobs: + security-checkpoint: + name: Check CI Clearance + runs-on: ubuntu-latest + permissions: + pull-requests: write + actions: write + if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' + steps: + - name: Comment on new Fork PR + if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 + uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 + with: + message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Please note that any changes to the workflow file will not be reflected in the run. + + - name: Comment on dependabot PR + if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id == 49699333 + uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 + with: + message: Set the milestone to the next minor version, check for supply chain attacks, and then add the `CI Cleared` label to allow CI to run. + + - name: "Remove Stale 'CI Cleared' Label" + if: github.event.action == 'synchronize' || github.event.action == 'reopened' + uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 + with: + labels: CI Cleared + + - name: "Remove 'CI Approval Required' Label" + if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && contains(github.event.pull_request.labels.*.name, 'CI Cleared')) + uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 + with: + labels: CI Approval Required + + - name: "Add 'CI Approval Required' Label" + if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) + uses: actions-ecosystem/action-add-labels@bd52874380e3909a1ac983768df6976535ece7f8 + with: + labels: CI Approval Required + github_token: ${{ github.token }} + + - name: Fail if PR has Unlabeled new Commits from User + if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) + run: exit 1 + + - name: Dispatch CI Pipeline Workflow + uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 + with: + dispatch-method: workflow_dispatch + owner: ${{ github.repository_owner }} + repo: ${{ github.event.pull_request.base.repo.name }} + ref: ${{ github.event.pull_request.base.ref }} + workflow: ci-pipeline.yml + token: ${{ github.token }} + workflow-inputs: | + { + "pull_request_number": "${{ github.event.pull_request.number }}" + } diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index d9f5981df8c..b5841dfd033 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -60,7 +60,6 @@ static async Task Main(string[] args) var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase); - var ciCompletionCheck = versionString.Equals("--ci-completion-check", StringComparison.OrdinalIgnoreCase); var genToken = versionString.Equals("--token-output-file", StringComparison.OrdinalIgnoreCase); if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) @@ -69,7 +68,6 @@ static async Task Main(string[] args) && !shaCheck && !fullNotes && !nuget - && !ciCompletionCheck && !genToken) { Console.WriteLine("Invalid version: " + versionString); @@ -151,17 +149,6 @@ static async Task Main(string[] args) return await Winget(client, actionsUrl, null); } - if (ciCompletionCheck) - { - if (args.Length < 3) - { - Console.WriteLine("Missing SHA or PEM Base64 for creating check run!"); - return 4543; - } - - return await CICompletionCheck(client, args[1], args[2]); - } - if (genToken) { @@ -1676,21 +1663,6 @@ static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string gitHubClient.Credentials = new Credentials(installToken.Token); } - static async ValueTask CICompletionCheck(GitHubClient gitHubClient, string currentSha, string pemBase64) - { - await GenerateAppCredentials(gitHubClient, pemBase64, false); - - await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Completion", currentSha) - { - CompletedAt = DateTime.UtcNow, - Conclusion = CheckConclusion.Success, - Output = new NewCheckRunOutput("CI Completion", "The CI Pipeline completed successfully"), - Status = CheckStatus.Completed, - }); - - return 0; - } - static void DebugAssert(bool condition, string message = null) { // This exists because one of the fucking asserts evaluates an enumerable or something and it was getting optimized out in release From 182d0f5dedb60bba4db6930d83f82d333a8b6443 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 13:07:38 -0400 Subject: [PATCH 046/111] Fix parsing TGS version in debian build --- .github/workflows/ci-pipeline.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 822e137cc6b..0f66cb3177a 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1256,10 +1256,6 @@ jobs: with: ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" - - name: Parse TGS version - run: | - echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV - - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml @@ -1267,15 +1263,22 @@ jobs: run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Execute Build Script (Unsigned) - if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev'))) + if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master')) run: sudo -E build/package/deb/build_package.sh - name: Execute Build Script (Signed) - if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) + if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master') env: PACKAGING_KEYGRIP: ${{ vars.PACKAGING_KEYGRIP }} + run: sudo -E build/package/deb/build_package.sh + + - name: Parse TGS version run: | - sudo -E build/package/deb/build_package.sh + echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV + + - name: Verify Package Files are Signed + if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master') + run: gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1.dsc gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1_amd64.changes gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1_amd64.buildinfo From e9d0fe1d320c08c05308f263deb8cabde1b873e6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 13:20:16 -0400 Subject: [PATCH 047/111] CI Pipeline can only be triggered via dispatch for PRs now --- .github/workflows/ci-pipeline.yml | 36 +++++++++++++---------------- .github/workflows/ci-security.yml | 38 +++++++++++++++++++------------ 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b3c1a9f0847..6cfb58ff5b2 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -25,10 +25,6 @@ on: branches: - dev - master - pull_request: - branches: - - dev - - master workflow_dispatch: inputs: pull_request_number: @@ -46,7 +42,7 @@ env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} concurrency: - group: "ci-${{ github.head_ref || (github.event_name != 'push' && github.event_name != 'schedule' && github.event.inputs.pull_request_number) || github.run_id }}-${{ github.event_name }}" + group: "ci-${{ (github.event_name != 'push' && github.event_name != 'schedule' && github.event.inputs.pull_request_number) || github.run_id }}-${{ github.event_name }}" cancel-in-progress: true jobs: @@ -71,7 +67,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Initialize CodeQL uses: github/codeql-action/init@v3 @@ -145,7 +141,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Build DMAPI Test Project run: | @@ -197,7 +193,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Checkout OpenDream run: | @@ -239,7 +235,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Retrieve dotnet-ef Tool Version id: dotnet-ef-tool @@ -301,7 +297,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -369,7 +365,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Build Docker Image run: docker build . -f build/Dockerfile @@ -405,7 +401,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -454,7 +450,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -569,7 +565,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -774,7 +770,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore @@ -852,7 +848,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Retrieve OpenAPI Spec uses: actions/download-artifact@v4 @@ -876,7 +872,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Retrieve Linux Unit Test Coverage (Debug) uses: actions/download-artifact@v4 @@ -1162,7 +1158,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Parse TGS version run: | @@ -1243,7 +1239,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Restore Wix dotnet Tool run: | @@ -1373,7 +1369,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event_name == 'pull_request' && github.event.number || github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - name: Restore run: dotnet restore diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 3a860fb5a9f..456abfcec29 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -1,6 +1,10 @@ name: 'CI Security' on: + pull_request: + branches: + - dev + - master pull_request_target: types: [ opened, reopened, labeled, synchronize ] branches: @@ -17,7 +21,6 @@ jobs: runs-on: ubuntu-latest permissions: pull-requests: write - actions: write if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' steps: - name: Comment on new Fork PR @@ -55,16 +58,23 @@ jobs: if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) run: exit 1 - - name: Dispatch CI Pipeline Workflow - uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 - with: - dispatch-method: workflow_dispatch - owner: ${{ github.repository_owner }} - repo: ${{ github.event.pull_request.base.repo.name }} - ref: ${{ github.event.pull_request.base.ref }} - workflow: ci-pipeline.yml - token: ${{ github.token }} - workflow-inputs: | - { - "pull_request_number": "${{ github.event.pull_request.number }}" - } + ci-dispatch: + name: Start CI Pipeline + runs-on: ubuntu-latest + needs: security-checkpoint + permissions: + actions: write + steps: + - name: Send Workflow Dispatch + uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 + with: + dispatch-method: workflow_dispatch + owner: ${{ github.repository_owner }} + repo: ${{ github.event.pull_request.base.repo.name }} + ref: ${{ github.event.pull_request.base.ref }} + workflow: ci-pipeline.yml + token: ${{ github.token }} + workflow-inputs: | + { + "pull_request_number": "${{ github.event.pull_request.number }}" + } From c212a44c15106dbfbf1d0f989f624e5fb64deb98 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 13:21:16 -0400 Subject: [PATCH 048/111] Fix concurrency issues with mirror sync --- .github/workflows/update-ss13-org-mirror.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/update-ss13-org-mirror.yml b/.github/workflows/update-ss13-org-mirror.yml index c5db334d7f5..cbac03119b1 100644 --- a/.github/workflows/update-ss13-org-mirror.yml +++ b/.github/workflows/update-ss13-org-mirror.yml @@ -12,6 +12,10 @@ env: TGS_DOTNET_VERSION: 8 TGS_DOTNET_QUALITY: ga +concurrency: + group: "ss13-mirror-sync" + cancel-in-progress: true + jobs: fork-sync: name: Fork Sync From f9382f1e040216b8d552e171bee5ecad51a1d6fe Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 13:29:02 -0400 Subject: [PATCH 049/111] Get rid of more unnecessary actions checks --- .github/workflows/ci-security.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 456abfcec29..fa2561c7426 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -1,10 +1,6 @@ name: 'CI Security' on: - pull_request: - branches: - - dev - - master pull_request_target: types: [ opened, reopened, labeled, synchronize ] branches: @@ -21,7 +17,7 @@ jobs: runs-on: ubuntu-latest permissions: pull-requests: write - if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' + if: (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' steps: - name: Comment on new Fork PR if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 From 3477871640d7fde7dcf9bb166ca3dbfccf76d485 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 14:51:20 -0400 Subject: [PATCH 050/111] Run CI on the merge commit --- .github/workflows/ci-security.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index fa2561c7426..b048468dc2e 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -23,7 +23,7 @@ jobs: if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 with: - message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Please note that any changes to the workflow file will not be reflected in the run. + message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. - name: Comment on dependabot PR if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id == 49699333 @@ -67,7 +67,7 @@ jobs: dispatch-method: workflow_dispatch owner: ${{ github.repository_owner }} repo: ${{ github.event.pull_request.base.repo.name }} - ref: ${{ github.event.pull_request.base.ref }} + ref: "refs/pull/${{ github.event.pull_request.number }}/merge" workflow: ci-pipeline.yml token: ${{ github.token }} workflow-inputs: | From e286fd016d5d21a13e38e6bcb0ddf1a9caeb5ba0 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 16:04:31 -0400 Subject: [PATCH 051/111] We are not BYOND exclusive anymore --- build/package/deb/debian/control | 4 ++-- src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/package/deb/debian/control b/build/package/deb/debian/control index f0a7ecbbe3f..67356359b6e 100644 --- a/build/package/deb/debian/control +++ b/build/package/deb/debian/control @@ -24,5 +24,5 @@ Depends: Recommends: libsystemd0, gdb, -Description: A production scale tool for BYOND server management - This is a toolset to manage production BYOND servers. It includes the ability to update the server without having to stop or shutdown the server (the update will take effect on a "reboot" of the server), the ability to start the server and restart it if it crashes, as well as systems for managing code and game files, and locally merging GitHub Pull Requests for test deployments. +Description: A production scale tool for DreamMaker server management + This is a toolset to manage production DreamMaker servers. It includes the ability to update the server without having to stop or shutdown the server (the update will take effect on a "reboot" of the server), the ability to start the server and restart it if it crashes, as well as systems for managing code and game files, and locally merging GitHub Pull Requests for test deployments. diff --git a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs index a700ccc9cba..dc15d9db687 100644 --- a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs +++ b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs @@ -76,7 +76,7 @@ public static void Configure(SwaggerGenOptions swaggerGenOptions, string assembl Name = "/tg/station 13", Url = new Uri("https://github.com/tgstation"), }, - Description = "A production scale tool for BYOND server management", + Description = "A production scale tool for DreamMaker server management", }); // Important to do this before applying our own filters From 68349dc91f6f2444cb83ed7b4699059d5383ccf3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 16:06:30 -0400 Subject: [PATCH 052/111] Specify classic GitHub tokens --- README.md | 2 +- docs/Features.dox | 2 +- src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs | 2 +- src/Tgstation.Server.Host/Setup/SetupWizard.cs | 2 +- src/Tgstation.Server.Host/appsettings.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 794543dc854..58b341d571a 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will - `General:InstanceLimit`: Maximum number of instances that may be created -- `General:GitHubAccessToken`: Specify a GitHub personal access token with no scopes here to highly mitigate the possiblity of 429 response codes from GitHub requests +- `General:GitHubAccessToken`: Specify a classic GitHub personal access token with no scopes here to highly mitigate the possiblity of 429 response codes from GitHub requests - `General:SkipAddingByondFirewallException`: Set to `true` if you have Windows firewall disabled diff --git a/docs/Features.dox b/docs/Features.dox index 64eca5a3f7c..6600cb295f7 100644 --- a/docs/Features.dox +++ b/docs/Features.dox @@ -5,7 +5,7 @@ @section features_list Comprehensive Feature List -tgstation-server is a BYOND server managment suite. It includes all the following features +tgstation-server is a DreamMaker server managment suite. It includes all the following features - Standalone server with OpenAPI 3.0 defined HTTP REST API - Web based client included diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index bc9c11e2f36..1790316e370 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -89,7 +89,7 @@ public sealed class GeneralConfiguration : ServerInformationBase public ushort ApiPort { get; set; } /// - /// A GitHub personal access token to use for bypassing rate limits on requests. Requires no scopes. + /// A classic GitHub personal access token to use for bypassing rate limits on requests. Requires no scopes. /// public string? GitHubAccessToken { get; set; } diff --git a/src/Tgstation.Server.Host/Setup/SetupWizard.cs b/src/Tgstation.Server.Host/Setup/SetupWizard.cs index 04cc54a3bfd..5c77ecffebc 100644 --- a/src/Tgstation.Server.Host/Setup/SetupWizard.cs +++ b/src/Tgstation.Server.Host/Setup/SetupWizard.cs @@ -705,7 +705,7 @@ async ValueTask ConfigureGeneral(CancellationToken cancell while (true); await console.WriteAsync(null, true, cancellationToken); - await console.WriteAsync("Enter a GitHub personal access token to bypass some rate limits (this is optional and does not require any scopes)", true, cancellationToken); + await console.WriteAsync("Enter a classic GitHub personal access token to bypass some rate limits (this is optional and does not require any scopes)", true, cancellationToken); await console.WriteAsync("GitHub personal access token: ", false, cancellationToken); newGeneralConfiguration.GitHubAccessToken = await console.ReadLineAsync(true, cancellationToken); if (String.IsNullOrWhiteSpace(newGeneralConfiguration.GitHubAccessToken)) diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 526f38ab386..732a61e54b2 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -3,7 +3,7 @@ General: # ConfigVersion: # Basic semver. Differs from TGS version to version. See changelog for current version MinimumPasswordLength: 15 # Minimum TGS user password length - GitHubAccessToken: # GitHub personal access token with no scopes used to bypass rate-limits + GitHubAccessToken: # A classic GitHub personal access token with no scopes used to bypass rate-limits SetupWizardMode: AutoDetect # If the interactive TGS setup wizard should run ByondTopicTimeout: 5000 # Timeout for BYOND /world/Topic() calls in milliseconds RestartTimeoutMinutes: 1 # Timeout for server restarts after requested by SIGTERM or the HTTP API From 7a66748ef2c7732bc974fe4afaedbe54f12e9e5d Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 17:28:17 -0400 Subject: [PATCH 053/111] Fix dispatch ref --- .github/workflows/ci-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index b048468dc2e..6aa0e7085b7 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -67,7 +67,7 @@ jobs: dispatch-method: workflow_dispatch owner: ${{ github.repository_owner }} repo: ${{ github.event.pull_request.base.repo.name }} - ref: "refs/pull/${{ github.event.pull_request.number }}/merge" + ref: ${{ github.event.pull_request.head.ref }} workflow: ci-pipeline.yml token: ${{ github.token }} workflow-inputs: | From 29a5e5b29f2b1ef065456340568ce5844b165383 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 17:50:37 -0400 Subject: [PATCH 054/111] Try to get the dispatched workflow statuses to appear on the original PR --- .github/workflows/ci-security.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 6aa0e7085b7..f7d27b317bf 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -23,7 +23,7 @@ jobs: if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 with: - message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. + message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Note that any changes to ci-security.yml will not be reflected in the run. - name: Comment on dependabot PR if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id == 49699333 @@ -60,17 +60,30 @@ jobs: needs: security-checkpoint permissions: actions: write + contents: write steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: "refs/pull/${{ github.event.pull_request.number }}/merge" + + - name: Generate Temporary Branch to Reference Merge + run: git push -f origin ${{ github.event.pull_request.number }}-merge + - name: Send Workflow Dispatch uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 with: dispatch-method: workflow_dispatch owner: ${{ github.repository_owner }} repo: ${{ github.event.pull_request.base.repo.name }} - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ github.event.pull_request.number }}-merge workflow: ci-pipeline.yml token: ${{ github.token }} workflow-inputs: | { "pull_request_number": "${{ github.event.pull_request.number }}" } + + - name: Delete Temporary Branch + if: always() + run: git push -d origin ${{ github.event.pull_request.number }}-merge From a70d7483a076350e6d6a1e6e8fdbde91c7a5151a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 18:27:45 -0400 Subject: [PATCH 055/111] Fix security temp branch push not working --- .github/workflows/ci-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index f7d27b317bf..f3e7fa4f8b9 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -68,7 +68,7 @@ jobs: ref: "refs/pull/${{ github.event.pull_request.number }}/merge" - name: Generate Temporary Branch to Reference Merge - run: git push -f origin ${{ github.event.pull_request.number }}-merge + run: git push -f -u origin ${{ github.event.pull_request.number }}-merge - name: Send Workflow Dispatch uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 From 135f665cd219adc8f28ec66e8ac766812793a9ef Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 18:38:09 -0400 Subject: [PATCH 056/111] Fine just fucking branch first --- .github/workflows/ci-security.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index f3e7fa4f8b9..407346915aa 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -65,10 +65,12 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - ref: "refs/pull/${{ github.event.pull_request.number }}/merge" + ref: refs/pull/${{ github.event.pull_request.number }}/merge - name: Generate Temporary Branch to Reference Merge - run: git push -f -u origin ${{ github.event.pull_request.number }}-merge + run: | + git checkout -b ${{ github.event.pull_request.number }}-merge + git push -f -u origin ${{ github.event.pull_request.number }}-merge - name: Send Workflow Dispatch uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 From 3dc2b19a93e4da0d37b8c5f78e9ca0b0c5bed961 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 19:07:36 -0400 Subject: [PATCH 057/111] Format ReleaseNotes Program.cs --- .../Tgstation.Server.ReleaseNotes/Program.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index b5841dfd033..0ea358d3992 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1,7 +1,6 @@ // This program is minimal effort and should be sent to remedial school using System; -using System.Buffers.Text; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; @@ -170,7 +169,7 @@ static async Task Main(string[] args) if (shaCheck) { - if(args.Length < 2) + if (args.Length < 2) { Console.WriteLine("Missing SHA for PR template!"); return 32; @@ -752,7 +751,7 @@ async Task CommitNotes(Component component, List notes) } if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal) || trimmedLine.StartsWith("/🆑", StringComparison.Ordinal)) { - if(!Enum.TryParse(targetComponent, out var component)) + if (!Enum.TryParse(targetComponent, out var component)) component = targetComponent.ToUpperInvariant() switch { "**CONFIGURATION**" or "CONFIGURATION" or "CONFIG" => Component.Configuration, @@ -878,7 +877,7 @@ The user account that created this pull request is available to correct any issu }); var prToModify = userPrsOnWingetRepo.Items.OrderByDescending(pr => pr.Number).FirstOrDefault(); - if(prToModify == null) + if (prToModify == null) { Console.WriteLine("Could not find open winget-pkgs PR!"); return 31; @@ -1001,7 +1000,7 @@ async Task RunPRs() .GroupBy(kvp => kvp.Key) .Select(grouping => new KeyValuePair(grouping.Key, grouping.Max(kvp => kvp.Value)))); - foreach(var maxVersionKvp in prResults.SelectMany(x => x.Item1) + foreach (var maxVersionKvp in prResults.SelectMany(x => x.Item1) .Where(x => !releasedComponentVersions.ContainsKey(x.Key)) .GroupBy(x => x.Key) .Select(group => { @@ -1027,7 +1026,7 @@ async Task RunPRs() var component = componentKvp.Key; var list = new List(); - foreach(var changelistDict in prResults.Select(x => x.Item1)) + foreach (var changelistDict in prResults.Select(x => x.Item1)) { if (!changelistDict.TryGetValue(component, out var changelist)) continue; @@ -1131,7 +1130,7 @@ static async Task FullNotes(IGitHubClient client) return 0; } - static readonly HttpClient httpClient = new ( + static readonly HttpClient httpClient = new( new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate @@ -1180,7 +1179,8 @@ static async Task GenerateNotes(IGitHubClient client, Dictionary release.Id); var milestones = await TripleCheckGitHubPagination( - apiOptions => client.Issue.Milestone.GetAllForRepository(RepoOwner, RepoName, new MilestoneRequest { + apiOptions => client.Issue.Milestone.GetAllForRepository(RepoOwner, RepoName, new MilestoneRequest + { State = ItemStateFilter.All }, apiOptions), milestone => milestone.Id); @@ -1388,7 +1388,7 @@ static string GenerateComponentNotes(ReleaseNotes releaseNotes, Component compon PrintChanges(newNotes, relevantChangelog); } - if(component == Component.DreamMakerApi) + if (component == Component.DreamMakerApi) { newNotes.AppendLine(); newNotes.AppendLine("#tgs-dmapi-release"); @@ -1438,7 +1438,7 @@ static async Task ReleaseNuget(IGitHubClient client) { Component.NugetClient, "Client" }, }; - foreach(var kvp in csprojNameMap) + foreach (var kvp in csprojNameMap) { var component = kvp.Key; var csprojPath = CsprojSubstitution.Replace("$PROJECT$", kvp.Value); @@ -1576,7 +1576,7 @@ [optional blank line(s), stripped] builder.AppendLine(); builder.Append(" * The following changes are for "); builder.Append(GetComponentDisplayName(kvp.Key, true)); - if(kvp.Key == Component.Configuration) + if (kvp.Key == Component.Configuration) { builder.Append(". You "); if (kvp.Value.Version.Minor == 0 && kvp.Value.Version.Build == 0) From 1b91fee77e42b616dc67c4ff34653b1f16f934e2 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 20:27:24 -0400 Subject: [PATCH 058/111] Rejigger the CI Pipeline again - Pass through expected merge SHA from security workflow for security confirmation. - Rely on one check from ReleaseNotes, `always()` updated in completion gate. - Cache ReleaseNotes build throughout pipeline. There's literally no way this will work as expected --- .github/workflows/ci-pipeline.yml | 590 +++++++++++++++--- .github/workflows/ci-security.yml | 78 ++- .../Tgstation.Server.ReleaseNotes/Program.cs | 79 +++ 3 files changed, 621 insertions(+), 126 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 8704b477d8a..b1cf211e635 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -30,6 +30,9 @@ on: pull_request_number: description: 'Pull Request Number' required: true + pull_request_current_merge_sha: + description: 'Pull Request Merge SHA' + required: true env: TGS_DOTNET_VERSION: 8 @@ -47,8 +50,68 @@ concurrency: cancel-in-progress: true jobs: + ci-start-gate: + name: CI Start Gate + runs-on: ubuntu-latest + steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Checkout (Branch) + uses: actions/checkout@v4 + if: github.event_name == 'push' || github.event_name == 'schedule' + + - name: Checkout (PR Merge) + uses: actions/checkout@v4 + if: github.event_name != 'push' && github.event_name != 'schedule' + with: + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } + + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Store ReleaseNotes Binaries + uses: actions/upload-artifact@v4 + with: + name: release_notes_bins + path: ./release_notes_bins/ + + - name: Set CI Check Run (Started) + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} + code-scanning: name: Code Scanning + needs: ci-start-gate runs-on: ubuntu-latest permissions: security-events: write @@ -70,7 +133,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Initialize CodeQL uses: github/codeql-action/init@v3 @@ -94,6 +181,7 @@ jobs: dmapi-build: name: Build DMAPI + needs: ci-start-gate strategy: fail-fast: false matrix: @@ -151,7 +239,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Build DMAPI Test Project run: | @@ -175,6 +287,7 @@ jobs: opendream-build: name: Build DMAPI (OpenDream) + needs: ci-start-gate strategy: fail-fast: false matrix: @@ -203,7 +316,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Checkout OpenDream run: | @@ -235,6 +372,7 @@ jobs: efcore-version-match: name: Check Nuget Versions Match Tools + needs: ci-start-gate runs-on: ubuntu-latest steps: - name: Checkout (Branch) @@ -245,7 +383,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Retrieve dotnet-ef Tool Version id: dotnet-ef-tool @@ -291,6 +453,7 @@ jobs: pages-build: name: Build gh-pages + needs: ci-start-gate runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -299,33 +462,23 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Checkout (Branch) - uses: actions/checkout@v4 - if: github.event_name == 'push' || github.event_name == 'schedule' - - - name: Checkout (PR Merge) - uses: actions/checkout@v4 - if: github.event_name != 'push' && github.event_name != 'schedule' - with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: gh-pages Clone run: git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Build Changelog (Incremental) run: | mv $HOME/tgsdox/changelog.yml ./ 2>/dev/null - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --generate-full-notes + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --generate-full-notes - name: Generate App Token run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV rm ${{ runner.temp }}/installation_secret.txt @@ -365,6 +518,7 @@ jobs: docker-build: name: Build Docker Image + needs: ci-start-gate runs-on: ubuntu-latest env: TGS_TELEMETRY_KEY_FILE: tgs_telemetry_key.txt @@ -377,7 +531,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Setup Telemetry Key File shell: bash @@ -427,7 +605,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Restore run: dotnet restore @@ -492,7 +694,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Restore run: dotnet restore @@ -626,7 +852,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Restore run: dotnet restore @@ -850,7 +1100,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Restore run: dotnet restore @@ -938,7 +1212,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Retrieve OpenAPI Spec uses: actions/download-artifact@v4 @@ -962,7 +1260,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Retrieve Linux Unit Test Coverage (Debug) uses: actions/download-artifact@v4 @@ -1206,6 +1528,7 @@ jobs: build-deb: name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595 + needs: ci-start-gate runs-on: ubuntu-latest env: TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt @@ -1250,7 +1573,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml @@ -1320,6 +1667,7 @@ jobs: build-msi: name: Build Windows Installer .exe + needs: ci-start-gate runs-on: windows-latest env: TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt @@ -1348,7 +1696,31 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } - name: Restore Wix dotnet Tool run: | @@ -1468,6 +1840,7 @@ jobs: check-winget-pr-template: name: Check winget-pkgs Pull Request Template is up to date + needs: ci-start-gate runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -1482,38 +1855,54 @@ jobs: curl -L -u "${{ vars.DEV_PUSH_USERNAME }}:${{ secrets.DEV_PUSH_TOKEN }}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" -o commits.json https://api.github.com/repos/microsoft/winget-pkgs/commits?path=.github/PULL_REQUEST_TEMPLATE.md echo "pr_template_sha=$(cat commits.json | jq '.[0].sha')" >> $GITHUB_OUTPUT - - name: Checkout (Branch) - uses: actions/checkout@v4 - if: github.event_name == 'push' || github.event_name == 'schedule' - - - name: Checkout (PR Merge) - uses: actions/checkout@v4 - if: github.event_name != 'push' && github.event_name != 'schedule' + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 with: - ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge" - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + name: release_notes_bins + path: release_notes_bins - name: Run ReleaseNotes Check - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --winget-template-check ${{ steps.get-sha.outputs.pr_template_sha }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --winget-template-check ${{ steps.get-sha.outputs.pr_template_sha }} - ci-completion-gate: # This job exists so there isn't a moving target for branch protections + ci-completion-gate: name: CI Completion Gate needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match ] runs-on: ubuntu-latest + if: always() steps: - - name: GitHub Requires at Least One Step for a Job - run: exit 0 + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + + - name: Update CI Check Run (Cancelled) + if: cancelled() + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} + + - name: Update CI Check Run (Failure) + if: !cancelled() && failure() + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} + + - name: Update CI Check Run (Success) + if: !cancelled() && failure() + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} + + - name: Fail Job if Prerequisites Failed + if: failure() + run: exit 1 deployment-gate: name: Deployment Start Gate needs: ci-completion-gate runs-on: ubuntu-latest - if: github.event_name == 'push' && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev') + if: github.event_name == 'push' steps: - name: GitHub Requires at Least One Step for a Job run: exit 0 @@ -1533,12 +1922,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Parse API version shell: powershell run: | @@ -1558,13 +1941,19 @@ jobs: $ProgressPreference = 'SilentlyContinue' Invoke-WebRequest -Uri https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -OutFile changelog.yml + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_API_VERSION }} --httpapi + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll ${{ env.TGS_API_VERSION }} --httpapi - name: Generate App Token shell: powershell run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} $installSecret = Get-Content ${{ runner.temp }}/installation_secret.txt echo "INSTALLATION_TOKEN=$installSecret" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append rm ${{ runner.temp }}/installation_secret.txt @@ -1605,12 +1994,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Parse DMAPI version shell: powershell run: | @@ -1629,13 +2012,19 @@ jobs: $ProgressPreference = 'SilentlyContinue' Invoke-WebRequest -Uri https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -OutFile changelog.yml + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_DM_VERSION }} --dmapi + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll ${{ env.TGS_DM_VERSION }} --dmapi - name: Generate App Token shell: powershell run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} $installSecret = Get-Content ${{ runner.temp }}/installation_secret.txt echo "INSTALLATION_TOKEN=$installSecret" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append rm ${{ runner.temp }}/installation_secret.txt @@ -1679,14 +2068,17 @@ jobs: - name: Restore run: dotnet restore - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --nuget + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --nuget - name: Publish Tgstation.Server.Common to NuGet uses: alirezanet/publish-nuget@e276c40afeb2a154046f0997820f2a9ea74832d9 # v3.1.0 @@ -1724,17 +2116,14 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Checkout - uses: actions/checkout@v4 - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins - name: Run ReleaseNotes with --ensure-release - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ensure-release ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ensure-release ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} deploy-tgs: name: Deploy TGS @@ -1784,9 +2173,6 @@ jobs: - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj - - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Prepare Artifacts shell: powershell run: build/package/winget/prepare_installer_input_artifacts.ps1 @@ -1871,13 +2257,19 @@ jobs: &"C:/Program Files/7-Zip/7z.exe" a ServerConsole.zip ./ServerConsole/* -tzip &"C:/Program Files/7-Zip/7z.exe" a ServerUpdatePackage.zip ./ServerUpdatePackage/* -tzip + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Generate Release Notes - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_VERSION }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll ${{ env.TGS_VERSION }} - name: Generate App Token shell: powershell run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} $installSecret = Get-Content ${{ runner.temp }}/installation_secret.txt echo "INSTALLATION_TOKEN=$installSecret" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append rm ${{ runner.temp }}/installation_secret.txt @@ -1985,26 +2377,23 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Checkout - uses: actions/checkout@v4 - - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: gh-pages Clone run: git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Build Changelog (Incremental) run: | mv $HOME/tgsdox/changelog.yml ./ 2>/dev/null - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --generate-full-notes + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --generate-full-notes - name: Generate App Token run: | - dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} + dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --token-output-file ${{ runner.temp }}/installation_secret.txt ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} echo "INSTALLATION_TOKEN=$(cat ${{ runner.temp }}/installation_secret.txt)" >> $GITHUB_ENV rm ${{ runner.temp }}/installation_secret.txt @@ -2090,15 +2479,18 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Build ReleaseNotes - run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes - - name: Retrieve Server Service uses: actions/download-artifact@v4 with: name: packaging-windows-raw-msi path: artifacts + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Execute Push Script shell: powershell run: build/package/winget/push_manifest.ps1 @@ -2111,4 +2503,4 @@ jobs: - name: Run ReleaseNotes with --link-winget shell: powershell - run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --link-winget ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --link-winget ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 407346915aa..6fb53695fbc 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -6,6 +6,9 @@ on: branches: - dev - master +env: + TGS_DOTNET_VERSION: 8 + TGS_DOTNET_QUALITY: ga concurrency: group: "ci-security-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" @@ -62,30 +65,51 @@ jobs: actions: write contents: write steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: refs/pull/${{ github.event.pull_request.number }}/merge - - - name: Generate Temporary Branch to Reference Merge - run: | - git checkout -b ${{ github.event.pull_request.number }}-merge - git push -f -u origin ${{ github.event.pull_request.number }}-merge - - - name: Send Workflow Dispatch - uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 - with: - dispatch-method: workflow_dispatch - owner: ${{ github.repository_owner }} - repo: ${{ github.event.pull_request.base.repo.name }} - ref: ${{ github.event.pull_request.number }}-merge - workflow: ci-pipeline.yml - token: ${{ github.token }} - workflow-inputs: | - { - "pull_request_number": "${{ github.event.pull_request.number }}" - } - - - name: Delete Temporary Branch - if: always() - run: git push -d origin ${{ github.event.pull_request.number }}-merge + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.pull_request.number }}/merge + + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Read Current SHA + id: get-pr-sha + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Generate Temporary Branch to Reference Merge + run: | + git checkout -b ${{ github.event.pull_request.number }}-merge + git push -f -u origin ${{ github.event.pull_request.number }}-merge + + - name: Send Workflow Dispatch + uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 + id: dispatch + with: + dispatch-method: workflow_dispatch + owner: ${{ github.repository_owner }} + repo: ${{ github.event.pull_request.base.repo.name }} + ref: ${{ github.event.pull_request.number }}-merge + workflow: ci-pipeline.yml + token: ${{ github.token }} + workflow-inputs: | + { + "pull_request_number": "${{ github.event.pull_request.number }}" + "pull_request_current_merge_sha": "${{ steps.get-pr-sha.outputs.head_sha }}" + } + + - name: Set CI Check Run (Pending) + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ steps.get-pr-sha.outputs.head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Pending ${{ steps.dispatch.outputs.run-id }} + + - name: Delete Temporary Branch + if: always() + run: git push -d origin ${{ github.event.pull_request.number }}-merge diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index 0ea358d3992..47faffcecab 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -59,6 +59,7 @@ static async Task Main(string[] args) var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase); + var ciCheck = versionString.Equals("--ci-check", StringComparison.OrdinalIgnoreCase); var genToken = versionString.Equals("--token-output-file", StringComparison.OrdinalIgnoreCase); if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) @@ -67,6 +68,7 @@ static async Task Main(string[] args) && !shaCheck && !fullNotes && !nuget + && !ciCheck && !genToken) { Console.WriteLine("Invalid version: " + versionString); @@ -148,6 +150,17 @@ static async Task Main(string[] args) return await Winget(client, actionsUrl, null); } + if (ciCheck) + { + if (args.Length < 5) + { + Console.WriteLine("Missing check parameters!"); + return 4543; + } + + return await CICheck(client, args[1], args[2], Enum.Parse(args[3]), Int64.Parse(args[4])); + } + if (genToken) { @@ -1663,6 +1676,72 @@ static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string gitHubClient.Credentials = new Credentials(installToken.Token); } + enum CheckMode + { + Pending, + Started, + Cancelled, + Success, + Failure, + } + + static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSha, string pemBase64, CheckMode mode, long runID) + { + await GenerateAppCredentials(gitHubClient, pemBase64, false); + + switch (mode) + { + case CheckMode.Pending: + await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) + { + DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", + }); + break; + case CheckMode.Started: + var prChecks = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); + var theCheckWeWant = prChecks.CheckRuns.FirstOrDefault(x => x.App.Id == AppId); + if (theCheckWeWant != null) + { + await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate + { + Status = CheckStatus.InProgress, + StartedAt = DateTimeOffset.UtcNow, + }); + } + else + await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) + { + DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", + Status = CheckStatus.InProgress, + StartedAt = DateTimeOffset.UtcNow, + }); + + break; + case CheckMode.Cancelled: + case CheckMode.Failure: + case CheckMode.Success: + var conclusion = mode switch + { + CheckMode.Cancelled => CheckConclusion.Cancelled, + CheckMode.Failure => CheckConclusion.Failure, + CheckMode.Success => CheckConclusion.Success, + _ => throw new InvalidOperationException("Impossible"), + }; + + var prChecks2 = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); + var theCheckWeWant2 = prChecks2.CheckRuns.First(x => x.App.Id == AppId); + await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant2.Id, new CheckRunUpdate + { + Status = CheckStatus.Completed, + CompletedAt = DateTimeOffset.UtcNow, + Conclusion = conclusion, + }); + break; + } + + return 0; + } + static void DebugAssert(bool condition, string message = null) { // This exists because one of the fucking asserts evaluates an enumerable or something and it was getting optimized out in release From d1069cc38bb60c5eddfa2d040605bbcdcab89ac7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 20:28:25 -0400 Subject: [PATCH 059/111] Fix workflow syntax errors? --- .github/workflows/ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b1cf211e635..2a0ac44004d 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1887,11 +1887,11 @@ jobs: run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - name: Update CI Check Run (Failure) - if: !cancelled() && failure() + if: (!cancelled() && failure()) run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - name: Update CI Check Run (Success) - if: !cancelled() && failure() + if: (!cancelled() && failure()) run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - name: Fail Job if Prerequisites Failed From 80b8e5d5660a92c00fc4c7f614d19d68ea751eba Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 20:29:24 -0400 Subject: [PATCH 060/111] Fix unit test not depending on `ci-start-gate` --- .github/workflows/ci-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 2a0ac44004d..c65da107c12 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -570,6 +570,7 @@ jobs: linux-unit-tests: name: Linux Tests + needs: ci-start-gate strategy: fail-fast: false matrix: @@ -665,6 +666,7 @@ jobs: windows-unit-tests: name: Windows Tests + needs: ci-start-gate strategy: fail-fast: false matrix: From 8bad0cd2fee236a591dd60d220265b075f1b4234 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 20:34:37 -0400 Subject: [PATCH 061/111] Infer SHA from github context --- .github/workflows/ci-pipeline.yml | 39 ++++++++++++++----------------- .github/workflows/ci-security.yml | 7 +----- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index c65da107c12..7066b205293 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -30,9 +30,6 @@ on: pull_request_number: description: 'Pull Request Number' required: true - pull_request_current_merge_sha: - description: 'Pull Request Merge SHA' - required: true env: TGS_DOTNET_VERSION: 8 @@ -78,7 +75,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -107,7 +104,7 @@ jobs: path: ./release_notes_bins/ - name: Set CI Check Run (Started) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} code-scanning: name: Code Scanning @@ -143,7 +140,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -249,7 +246,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -326,7 +323,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -393,7 +390,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -541,7 +538,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -616,7 +613,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -706,7 +703,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -864,7 +861,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1112,7 +1109,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1224,7 +1221,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1272,7 +1269,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1585,7 +1582,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1708,7 +1705,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_current_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1886,15 +1883,15 @@ jobs: - name: Update CI Check Run (Cancelled) if: cancelled() - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - name: Update CI Check Run (Failure) if: (!cancelled() && failure()) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - name: Update CI Check Run (Success) if: (!cancelled() && failure()) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.inputs.pull_request_current_merge_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - name: Fail Job if Prerequisites Failed if: failure() diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 6fb53695fbc..7571f5c4c01 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -82,10 +82,6 @@ jobs: - name: Build ReleaseNotes run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Read Current SHA - id: get-pr-sha - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Generate Temporary Branch to Reference Merge run: | git checkout -b ${{ github.event.pull_request.number }}-merge @@ -104,11 +100,10 @@ jobs: workflow-inputs: | { "pull_request_number": "${{ github.event.pull_request.number }}" - "pull_request_current_merge_sha": "${{ steps.get-pr-sha.outputs.head_sha }}" } - name: Set CI Check Run (Pending) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ steps.get-pr-sha.outputs.head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Pending ${{ steps.dispatch.outputs.run-id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Pending ${{ steps.dispatch.outputs.run-id }} - name: Delete Temporary Branch if: always() From 18893bfe4e95271f394e9a4ac009cc1f9bde2253 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 20:49:32 -0400 Subject: [PATCH 062/111] Don't try to start the completion gate if the start gate never ran --- .github/workflows/ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 7066b205293..a30fa048b70 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1865,9 +1865,9 @@ jobs: ci-completion-gate: name: CI Completion Gate - needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match ] + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match, ci-start-gate ] runs-on: ubuntu-latest - if: always() + if: always() && needs.ci-start-gate.result == 'success' steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 From 3aabfa6b4e94147ed1674c57cdf7c5aeaf7bff4f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 21:06:22 -0400 Subject: [PATCH 063/111] Fix completion conditionals --- .github/workflows/ci-pipeline.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index a30fa048b70..70328c8710b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1882,19 +1882,19 @@ jobs: path: release_notes_bins - name: Update CI Check Run (Cancelled) - if: cancelled() - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} + if: needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled' + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - name: Update CI Check Run (Failure) - if: (!cancelled() && failure()) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} + if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - name: Update CI Check Run (Success) - if: (!cancelled() && failure()) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} + if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.code-scanning.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success' + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - name: Fail Job if Prerequisites Failed - if: failure() + if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) run: exit 1 deployment-gate: @@ -2368,7 +2368,6 @@ jobs: name: Regenerate Changelog runs-on: ubuntu-latest needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 From 50eaadd9e0806cce5002d73c1025e69bde75b829 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 21:25:20 -0400 Subject: [PATCH 064/111] Do not fail on missing release notes token if unused --- .../Tgstation.Server.ReleaseNotes/Program.cs | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index 47faffcecab..b1b83c4a496 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -110,20 +110,7 @@ static async Task Main(string[] args) break; } - const string ReleaseNotesEnvVar = "TGS_RELEASE_NOTES_TOKEN"; - var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar); - if (String.IsNullOrWhiteSpace(githubToken) && !doNotCloseMilestone && !ensureRelease) - { - Console.WriteLine("Missing " + ReleaseNotesEnvVar + " environment variable!"); - return 3; - } - var client = new GitHubClient(new Octokit.ProductHeaderValue("tgs_release_notes")); - if (!String.IsNullOrWhiteSpace(githubToken)) - { - client.Credentials = new Credentials(githubToken); - } - try { if (ensureRelease) @@ -139,17 +126,6 @@ static async Task Main(string[] args) return await EnsureRelease(client); } - if (linkWinget) - { - if (args.Length < 2 || !Uri.TryCreate(args[1], new UriCreationOptions(), out var actionsUrl)) - { - Console.WriteLine("Missing/Invalid actions URL!"); - return 30; - } - - return await Winget(client, actionsUrl, null); - } - if (ciCheck) { if (args.Length < 5) @@ -161,7 +137,6 @@ static async Task Main(string[] args) return await CICheck(client, args[1], args[2], Enum.Parse(args[3]), Int64.Parse(args[4])); } - if (genToken) { if (args.Length < 3) @@ -180,6 +155,30 @@ static async Task Main(string[] args) return 0; } + const string ReleaseNotesEnvVar = "TGS_RELEASE_NOTES_TOKEN"; + var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar); + if (String.IsNullOrWhiteSpace(githubToken) && !doNotCloseMilestone && !ensureRelease) + { + Console.WriteLine("Missing " + ReleaseNotesEnvVar + " environment variable!"); + return 3; + } + + if (!String.IsNullOrWhiteSpace(githubToken)) + { + client.Credentials = new Credentials(githubToken); + } + + if (linkWinget) + { + if (args.Length < 2 || !Uri.TryCreate(args[1], new UriCreationOptions(), out var actionsUrl)) + { + Console.WriteLine("Missing/Invalid actions URL!"); + return 30; + } + + return await Winget(client, actionsUrl, null); + } + if (shaCheck) { if (args.Length < 2) @@ -1684,7 +1683,7 @@ enum CheckMode Success, Failure, } - + static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSha, string pemBase64, CheckMode mode, long runID) { await GenerateAppCredentials(gitHubClient, pemBase64, false); From b497b047b04f3b813d2e5351c7ebc69201d77b28 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 21:30:36 -0400 Subject: [PATCH 065/111] Fix check run location calculation in security workflow --- .github/workflows/ci-security.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 7571f5c4c01..3ada9fae3f7 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -82,6 +82,10 @@ jobs: - name: Build ReleaseNotes run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Read Current SHA # Can't rely on github.sha as it's for the base branch + id: get-pr-sha + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Generate Temporary Branch to Reference Merge run: | git checkout -b ${{ github.event.pull_request.number }}-merge @@ -103,7 +107,7 @@ jobs: } - name: Set CI Check Run (Pending) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Pending ${{ steps.dispatch.outputs.run-id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ steps.get-pr-sha.outputs.head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Pending ${{ steps.dispatch.outputs.run-id }} - name: Delete Temporary Branch if: always() From 70b407174b3296c2fe3519a47c0fffb99aa44b4c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 22:14:27 -0400 Subject: [PATCH 066/111] Try running on the HEAD SHA instead of the merge in hopes of simplicity --- .github/workflows/ci-pipeline.yml | 85 ++++++------------- .github/workflows/ci-security.yml | 26 +++--- .../Tgstation.Server.ReleaseNotes/Program.cs | 79 ----------------- 3 files changed, 41 insertions(+), 149 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 70328c8710b..f3e82ef9aed 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -30,6 +30,9 @@ on: pull_request_number: description: 'Pull Request Number' required: true + pull_request_merge_sha: + description: 'Pull Request Merge SHA' + required: true env: TGS_DOTNET_VERSION: 8 @@ -47,8 +50,8 @@ concurrency: cancel-in-progress: true jobs: - ci-start-gate: - name: CI Start Gate + release-notes-build: + name: Build Release Notes for Other Jobs runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -75,7 +78,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -103,12 +106,8 @@ jobs: name: release_notes_bins path: ./release_notes_bins/ - - name: Set CI Check Run (Started) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} - code-scanning: name: Code Scanning - needs: ci-start-gate runs-on: ubuntu-latest permissions: security-events: write @@ -140,7 +139,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -178,7 +177,6 @@ jobs: dmapi-build: name: Build DMAPI - needs: ci-start-gate strategy: fail-fast: false matrix: @@ -246,7 +244,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -284,7 +282,6 @@ jobs: opendream-build: name: Build DMAPI (OpenDream) - needs: ci-start-gate strategy: fail-fast: false matrix: @@ -323,7 +320,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -369,7 +366,6 @@ jobs: efcore-version-match: name: Check Nuget Versions Match Tools - needs: ci-start-gate runs-on: ubuntu-latest steps: - name: Checkout (Branch) @@ -390,7 +386,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -450,7 +446,7 @@ jobs: pages-build: name: Build gh-pages - needs: ci-start-gate + needs: release-notes-build runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -515,7 +511,6 @@ jobs: docker-build: name: Build Docker Image - needs: ci-start-gate runs-on: ubuntu-latest env: TGS_TELEMETRY_KEY_FILE: tgs_telemetry_key.txt @@ -538,7 +533,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -567,7 +562,6 @@ jobs: linux-unit-tests: name: Linux Tests - needs: ci-start-gate strategy: fail-fast: false matrix: @@ -613,7 +607,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -663,7 +657,6 @@ jobs: windows-unit-tests: name: Windows Tests - needs: ci-start-gate strategy: fail-fast: false matrix: @@ -703,7 +696,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -861,7 +854,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1109,7 +1102,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1221,7 +1214,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1269,7 +1262,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1527,7 +1520,6 @@ jobs: build-deb: name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595 - needs: ci-start-gate runs-on: ubuntu-latest env: TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt @@ -1582,7 +1574,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1666,7 +1658,6 @@ jobs: build-msi: name: Build Windows Installer .exe - needs: ci-start-gate runs-on: windows-latest env: TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt @@ -1705,7 +1696,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1839,7 +1830,7 @@ jobs: check-winget-pr-template: name: Check winget-pkgs Pull Request Template is up to date - needs: ci-start-gate + needs: release-notes-build runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -1864,38 +1855,12 @@ jobs: run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --winget-template-check ${{ steps.get-sha.outputs.pr_template_sha }} ci-completion-gate: - name: CI Completion Gate - needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match, ci-start-gate ] + name: CI Completion Gate # Used as a branch protection ruleset target + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match ] runs-on: ubuntu-latest - if: always() && needs.ci-start-gate.result == 'success' steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Retrieve ReleaseNotes Binaries - uses: actions/download-artifact@v4 - with: - name: release_notes_bins - path: release_notes_bins - - - name: Update CI Check Run (Cancelled) - if: needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled' - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - - - name: Update CI Check Run (Failure) - if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - - - name: Update CI Check Run (Success) - if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.code-scanning.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success' - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - - - name: Fail Job if Prerequisites Failed - if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) - run: exit 1 + - name: Mandatory Empty Step + run: exit 0 deployment-gate: name: Deployment Start Gate diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 3ada9fae3f7..2e6926d5239 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -74,7 +74,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - ref: refs/pull/${{ github.event.pull_request.number }}/merge + ref: refs/pull/${{ github.event.pull_request.number }}/head - name: Restore run: dotnet restore @@ -82,14 +82,22 @@ jobs: - name: Build ReleaseNotes run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Checkout + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.pull_request.number }}/merge + path: merge_workspace + - name: Read Current SHA # Can't rely on github.sha as it's for the base branch id: get-pr-sha - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + run: | + cd merge_workspace + echo "merge_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Generate Temporary Branch to Reference Merge + - name: Generate Temporary Branch to Reference Head run: | - git checkout -b ${{ github.event.pull_request.number }}-merge - git push -f -u origin ${{ github.event.pull_request.number }}-merge + git checkout -b ${{ github.event.pull_request.number }}-head + git push -f -u origin ${{ github.event.pull_request.number }}-head - name: Send Workflow Dispatch uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 @@ -98,17 +106,15 @@ jobs: dispatch-method: workflow_dispatch owner: ${{ github.repository_owner }} repo: ${{ github.event.pull_request.base.repo.name }} - ref: ${{ github.event.pull_request.number }}-merge + ref: ${{ github.event.pull_request.number }}-head workflow: ci-pipeline.yml token: ${{ github.token }} workflow-inputs: | { "pull_request_number": "${{ github.event.pull_request.number }}" + "pull_request_merge_sha": "${{ steps.get-pr-sha.outputs.merge_sha }}" } - - name: Set CI Check Run (Pending) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ steps.get-pr-sha.outputs.head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Pending ${{ steps.dispatch.outputs.run-id }} - - name: Delete Temporary Branch if: always() - run: git push -d origin ${{ github.event.pull_request.number }}-merge + run: git push -d origin ${{ github.event.pull_request.number }}-head diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index b1b83c4a496..46c887fd5f6 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -59,7 +59,6 @@ static async Task Main(string[] args) var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase); - var ciCheck = versionString.Equals("--ci-check", StringComparison.OrdinalIgnoreCase); var genToken = versionString.Equals("--token-output-file", StringComparison.OrdinalIgnoreCase); if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) @@ -68,7 +67,6 @@ static async Task Main(string[] args) && !shaCheck && !fullNotes && !nuget - && !ciCheck && !genToken) { Console.WriteLine("Invalid version: " + versionString); @@ -126,17 +124,6 @@ static async Task Main(string[] args) return await EnsureRelease(client); } - if (ciCheck) - { - if (args.Length < 5) - { - Console.WriteLine("Missing check parameters!"); - return 4543; - } - - return await CICheck(client, args[1], args[2], Enum.Parse(args[3]), Int64.Parse(args[4])); - } - if (genToken) { if (args.Length < 3) @@ -1675,72 +1662,6 @@ static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string gitHubClient.Credentials = new Credentials(installToken.Token); } - enum CheckMode - { - Pending, - Started, - Cancelled, - Success, - Failure, - } - - static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSha, string pemBase64, CheckMode mode, long runID) - { - await GenerateAppCredentials(gitHubClient, pemBase64, false); - - switch (mode) - { - case CheckMode.Pending: - await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) - { - DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", - }); - break; - case CheckMode.Started: - var prChecks = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); - var theCheckWeWant = prChecks.CheckRuns.FirstOrDefault(x => x.App.Id == AppId); - if (theCheckWeWant != null) - { - await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate - { - Status = CheckStatus.InProgress, - StartedAt = DateTimeOffset.UtcNow, - }); - } - else - await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) - { - DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", - Status = CheckStatus.InProgress, - StartedAt = DateTimeOffset.UtcNow, - }); - - break; - case CheckMode.Cancelled: - case CheckMode.Failure: - case CheckMode.Success: - var conclusion = mode switch - { - CheckMode.Cancelled => CheckConclusion.Cancelled, - CheckMode.Failure => CheckConclusion.Failure, - CheckMode.Success => CheckConclusion.Success, - _ => throw new InvalidOperationException("Impossible"), - }; - - var prChecks2 = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); - var theCheckWeWant2 = prChecks2.CheckRuns.First(x => x.App.Id == AppId); - await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant2.Id, new CheckRunUpdate - { - Status = CheckStatus.Completed, - CompletedAt = DateTimeOffset.UtcNow, - Conclusion = conclusion, - }); - break; - } - - return 0; - } - static void DebugAssert(bool condition, string message = null) { // This exists because one of the fucking asserts evaluates an enumerable or something and it was getting optimized out in release From 6f8e129fcc5df3d267f4974b471a1f9c9a34c8c1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 22:19:35 -0400 Subject: [PATCH 067/111] More CI Fixes - Fix security dispatch JSON. - Remove unnecessary CI security steps. --- .github/workflows/ci-security.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 2e6926d5239..5b408b6b696 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -65,23 +65,11 @@ jobs: actions: write contents: write steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Checkout uses: actions/checkout@v4 with: ref: refs/pull/${{ github.event.pull_request.number }}/head - - name: Restore - run: dotnet restore - - - name: Build ReleaseNotes - run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Checkout uses: actions/checkout@v4 with: @@ -111,7 +99,7 @@ jobs: token: ${{ github.token }} workflow-inputs: | { - "pull_request_number": "${{ github.event.pull_request.number }}" + "pull_request_number": "${{ github.event.pull_request.number }}", "pull_request_merge_sha": "${{ steps.get-pr-sha.outputs.merge_sha }}" } From 99b319f97fdb25bfeb36446639f258b6be42ea72 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 22:33:48 -0400 Subject: [PATCH 068/111] Revert "More CI Fixes" This reverts commit 6f8e129fcc5df3d267f4974b471a1f9c9a34c8c1. --- .github/workflows/ci-security.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 5b408b6b696..2e6926d5239 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -65,11 +65,23 @@ jobs: actions: write contents: write steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Checkout uses: actions/checkout@v4 with: ref: refs/pull/${{ github.event.pull_request.number }}/head + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Checkout uses: actions/checkout@v4 with: @@ -99,7 +111,7 @@ jobs: token: ${{ github.token }} workflow-inputs: | { - "pull_request_number": "${{ github.event.pull_request.number }}", + "pull_request_number": "${{ github.event.pull_request.number }}" "pull_request_merge_sha": "${{ steps.get-pr-sha.outputs.merge_sha }}" } From f9667a2801ac14c222dfd3113765834ba3c59086 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 22:33:59 -0400 Subject: [PATCH 069/111] Revert "Try running on the HEAD SHA instead of the merge in hopes of simplicity" This reverts commit 70b407174b3296c2fe3519a47c0fffb99aa44b4c. --- .github/workflows/ci-pipeline.yml | 85 +++++++++++++------ .github/workflows/ci-security.yml | 26 +++--- .../Tgstation.Server.ReleaseNotes/Program.cs | 79 +++++++++++++++++ 3 files changed, 149 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index f3e82ef9aed..70328c8710b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -30,9 +30,6 @@ on: pull_request_number: description: 'Pull Request Number' required: true - pull_request_merge_sha: - description: 'Pull Request Merge SHA' - required: true env: TGS_DOTNET_VERSION: 8 @@ -50,8 +47,8 @@ concurrency: cancel-in-progress: true jobs: - release-notes-build: - name: Build Release Notes for Other Jobs + ci-start-gate: + name: CI Start Gate runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -78,7 +75,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -106,8 +103,12 @@ jobs: name: release_notes_bins path: ./release_notes_bins/ + - name: Set CI Check Run (Started) + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} + code-scanning: name: Code Scanning + needs: ci-start-gate runs-on: ubuntu-latest permissions: security-events: write @@ -139,7 +140,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -177,6 +178,7 @@ jobs: dmapi-build: name: Build DMAPI + needs: ci-start-gate strategy: fail-fast: false matrix: @@ -244,7 +246,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -282,6 +284,7 @@ jobs: opendream-build: name: Build DMAPI (OpenDream) + needs: ci-start-gate strategy: fail-fast: false matrix: @@ -320,7 +323,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -366,6 +369,7 @@ jobs: efcore-version-match: name: Check Nuget Versions Match Tools + needs: ci-start-gate runs-on: ubuntu-latest steps: - name: Checkout (Branch) @@ -386,7 +390,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -446,7 +450,7 @@ jobs: pages-build: name: Build gh-pages - needs: release-notes-build + needs: ci-start-gate runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -511,6 +515,7 @@ jobs: docker-build: name: Build Docker Image + needs: ci-start-gate runs-on: ubuntu-latest env: TGS_TELEMETRY_KEY_FILE: tgs_telemetry_key.txt @@ -533,7 +538,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -562,6 +567,7 @@ jobs: linux-unit-tests: name: Linux Tests + needs: ci-start-gate strategy: fail-fast: false matrix: @@ -607,7 +613,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -657,6 +663,7 @@ jobs: windows-unit-tests: name: Windows Tests + needs: ci-start-gate strategy: fail-fast: false matrix: @@ -696,7 +703,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -854,7 +861,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1102,7 +1109,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1214,7 +1221,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1262,7 +1269,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1520,6 +1527,7 @@ jobs: build-deb: name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595 + needs: ci-start-gate runs-on: ubuntu-latest env: TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt @@ -1574,7 +1582,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1658,6 +1666,7 @@ jobs: build-msi: name: Build Windows Installer .exe + needs: ci-start-gate runs-on: windows-latest env: TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt @@ -1696,7 +1705,7 @@ jobs: - name: Abort if PR Merge SHA has Changed uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != inputs.pull_request_merge_sha + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha with: script: | const delay = ms => new Promise(res => setTimeout(res, ms)); @@ -1830,7 +1839,7 @@ jobs: check-winget-pr-template: name: Check winget-pkgs Pull Request Template is up to date - needs: release-notes-build + needs: ci-start-gate runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -1855,12 +1864,38 @@ jobs: run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --winget-template-check ${{ steps.get-sha.outputs.pr_template_sha }} ci-completion-gate: - name: CI Completion Gate # Used as a branch protection ruleset target - needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match ] + name: CI Completion Gate + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match, ci-start-gate ] runs-on: ubuntu-latest + if: always() && needs.ci-start-gate.result == 'success' steps: - - name: Mandatory Empty Step - run: exit 0 + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + + - name: Update CI Check Run (Cancelled) + if: needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled' + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} + + - name: Update CI Check Run (Failure) + if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} + + - name: Update CI Check Run (Success) + if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.code-scanning.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success' + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} + + - name: Fail Job if Prerequisites Failed + if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) + run: exit 1 deployment-gate: name: Deployment Start Gate diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 2e6926d5239..3ada9fae3f7 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -74,7 +74,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - ref: refs/pull/${{ github.event.pull_request.number }}/head + ref: refs/pull/${{ github.event.pull_request.number }}/merge - name: Restore run: dotnet restore @@ -82,22 +82,14 @@ jobs: - name: Build ReleaseNotes run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Checkout - uses: actions/checkout@v4 - with: - ref: refs/pull/${{ github.event.pull_request.number }}/merge - path: merge_workspace - - name: Read Current SHA # Can't rely on github.sha as it's for the base branch id: get-pr-sha - run: | - cd merge_workspace - echo "merge_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Generate Temporary Branch to Reference Head + - name: Generate Temporary Branch to Reference Merge run: | - git checkout -b ${{ github.event.pull_request.number }}-head - git push -f -u origin ${{ github.event.pull_request.number }}-head + git checkout -b ${{ github.event.pull_request.number }}-merge + git push -f -u origin ${{ github.event.pull_request.number }}-merge - name: Send Workflow Dispatch uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 @@ -106,15 +98,17 @@ jobs: dispatch-method: workflow_dispatch owner: ${{ github.repository_owner }} repo: ${{ github.event.pull_request.base.repo.name }} - ref: ${{ github.event.pull_request.number }}-head + ref: ${{ github.event.pull_request.number }}-merge workflow: ci-pipeline.yml token: ${{ github.token }} workflow-inputs: | { "pull_request_number": "${{ github.event.pull_request.number }}" - "pull_request_merge_sha": "${{ steps.get-pr-sha.outputs.merge_sha }}" } + - name: Set CI Check Run (Pending) + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ steps.get-pr-sha.outputs.head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Pending ${{ steps.dispatch.outputs.run-id }} + - name: Delete Temporary Branch if: always() - run: git push -d origin ${{ github.event.pull_request.number }}-head + run: git push -d origin ${{ github.event.pull_request.number }}-merge diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index 46c887fd5f6..b1b83c4a496 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -59,6 +59,7 @@ static async Task Main(string[] args) var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase); + var ciCheck = versionString.Equals("--ci-check", StringComparison.OrdinalIgnoreCase); var genToken = versionString.Equals("--token-output-file", StringComparison.OrdinalIgnoreCase); if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) @@ -67,6 +68,7 @@ static async Task Main(string[] args) && !shaCheck && !fullNotes && !nuget + && !ciCheck && !genToken) { Console.WriteLine("Invalid version: " + versionString); @@ -124,6 +126,17 @@ static async Task Main(string[] args) return await EnsureRelease(client); } + if (ciCheck) + { + if (args.Length < 5) + { + Console.WriteLine("Missing check parameters!"); + return 4543; + } + + return await CICheck(client, args[1], args[2], Enum.Parse(args[3]), Int64.Parse(args[4])); + } + if (genToken) { if (args.Length < 3) @@ -1662,6 +1675,72 @@ static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string gitHubClient.Credentials = new Credentials(installToken.Token); } + enum CheckMode + { + Pending, + Started, + Cancelled, + Success, + Failure, + } + + static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSha, string pemBase64, CheckMode mode, long runID) + { + await GenerateAppCredentials(gitHubClient, pemBase64, false); + + switch (mode) + { + case CheckMode.Pending: + await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) + { + DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", + }); + break; + case CheckMode.Started: + var prChecks = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); + var theCheckWeWant = prChecks.CheckRuns.FirstOrDefault(x => x.App.Id == AppId); + if (theCheckWeWant != null) + { + await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate + { + Status = CheckStatus.InProgress, + StartedAt = DateTimeOffset.UtcNow, + }); + } + else + await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) + { + DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", + Status = CheckStatus.InProgress, + StartedAt = DateTimeOffset.UtcNow, + }); + + break; + case CheckMode.Cancelled: + case CheckMode.Failure: + case CheckMode.Success: + var conclusion = mode switch + { + CheckMode.Cancelled => CheckConclusion.Cancelled, + CheckMode.Failure => CheckConclusion.Failure, + CheckMode.Success => CheckConclusion.Success, + _ => throw new InvalidOperationException("Impossible"), + }; + + var prChecks2 = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); + var theCheckWeWant2 = prChecks2.CheckRuns.First(x => x.App.Id == AppId); + await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant2.Id, new CheckRunUpdate + { + Status = CheckStatus.Completed, + CompletedAt = DateTimeOffset.UtcNow, + Conclusion = conclusion, + }); + break; + } + + return 0; + } + static void DebugAssert(bool condition, string message = null) { // This exists because one of the fucking asserts evaluates an enumerable or something and it was getting optimized out in release From be08bf7c909d813fc048767c118a590017afe84e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 22:36:55 -0400 Subject: [PATCH 070/111] Create checks on PR HEAD commit to properly display in GitHub UI --- .github/workflows/ci-pipeline.yml | 11 +++++++---- .github/workflows/ci-security.yml | 9 +++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 70328c8710b..ab32822e21c 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -30,6 +30,9 @@ on: pull_request_number: description: 'Pull Request Number' required: true + pull_request_head_sha: + description: 'Pull Request HEAD SHA' + required: true env: TGS_DOTNET_VERSION: 8 @@ -104,7 +107,7 @@ jobs: path: ./release_notes_bins/ - name: Set CI Check Run (Started) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} code-scanning: name: Code Scanning @@ -1883,15 +1886,15 @@ jobs: - name: Update CI Check Run (Cancelled) if: needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled' - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - name: Update CI Check Run (Failure) if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - name: Update CI Check Run (Success) if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.code-scanning.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success' - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - name: Fail Job if Prerequisites Failed if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 3ada9fae3f7..2a49ee9d24f 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -82,10 +82,6 @@ jobs: - name: Build ReleaseNotes run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - name: Read Current SHA # Can't rely on github.sha as it's for the base branch - id: get-pr-sha - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Generate Temporary Branch to Reference Merge run: | git checkout -b ${{ github.event.pull_request.number }}-merge @@ -103,11 +99,12 @@ jobs: token: ${{ github.token }} workflow-inputs: | { - "pull_request_number": "${{ github.event.pull_request.number }}" + "pull_request_number": "${{ github.event.pull_request.number }}", + "pull_request_head": "${{ github.event.pull_request.head.sha }}" } - name: Set CI Check Run (Pending) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ steps.get-pr-sha.outputs.head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Pending ${{ steps.dispatch.outputs.run-id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.pull_request.head.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Pending ${{ steps.dispatch.outputs.run-id }} - name: Delete Temporary Branch if: always() From c5987590b56ec836a03f2294d0dd1af19adf9210 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 22:44:29 -0400 Subject: [PATCH 071/111] Be more lenient on flaky test failure --- .github/workflows/scripts/rerunFlakyTests.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/rerunFlakyTests.js b/.github/workflows/scripts/rerunFlakyTests.js index 5bacef12783..589d6d85c99 100644 --- a/.github/workflows/scripts/rerunFlakyTests.js +++ b/.github/workflows/scripts/rerunFlakyTests.js @@ -31,8 +31,8 @@ export async function rerunFlakyTests({ github, context }) { context.payload.workflow_run.run_attempt ); - if (failingJobs.length > 1) { - console.log("Multiple jobs failing. PROBABLY not flaky, not rerunning."); + if (failingJobs.length > 3) { + console.log("Many jobs failing. PROBABLY not flaky, not rerunning."); return; } @@ -40,8 +40,8 @@ export async function rerunFlakyTests({ github, context }) { console.log(`Failing job: ${job.name}`) return CONSIDERED_JOBS.some((title) => job.name.startsWith(title)); }); - if (filteredFailingJobs.length === 0) { - console.log("Failing jobs are NOT designated flaky. Not rerunning."); + if (filteredFailingJobs.length !== failingJobs.length) { + console.log("One or more failing jobs are NOT designated flaky. Not rerunning."); return; } From 414dc8b90cf6688b140b95cea6cf8adfd8be3812 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 22:49:48 -0400 Subject: [PATCH 072/111] Set check back to pending on re-runs --- .github/workflows/ci-pipeline.yml | 50 +++++++++++++------ .../Tgstation.Server.ReleaseNotes/Program.cs | 13 +++++ 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index ab32822e21c..ae3b13a9ebc 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -769,13 +769,6 @@ jobs: TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt runs-on: windows-latest steps: - - name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later - shell: powershell - if: ${{ matrix.database-type == 'SqlServer' }} - run: | - Write-Host "Checking" - sqlcmd -l 600 -S "(localdb)\MSSQLLocalDB" -Q "SELECT @@VERSION;" - - name: Setup dotnet uses: actions/setup-dotnet@v4 with: @@ -785,6 +778,22 @@ jobs: ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + + - name: Update CI Check Run (Rerun) # Do this here because these are the flakiest tests + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} + + - name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later + shell: powershell + if: ${{ matrix.database-type == 'SqlServer' }} + run: | + Write-Host "Checking" + sqlcmd -l 600 -S "(localdb)\MSSQLLocalDB" -Q "SELECT @@VERSION;" + - name: Setup Node.JS uses: actions/setup-node@v4 with: @@ -1042,15 +1051,6 @@ jobs: TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt runs-on: ubuntu-latest steps: - - name: Disable ptrace_scope - run: echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope - - - name: Install Native Dependencies - run: | - sudo dpkg --add-architecture i386 - sudo apt-get update - sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb libgcc-s1:i386 libgdiplus - - name: Setup dotnet uses: actions/setup-dotnet@v4 with: @@ -1060,6 +1060,24 @@ jobs: ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + + - name: Update CI Check Run (Rerun) # Do this here because these are the flakiest tests + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} + + - name: Disable ptrace_scope + run: echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + + - name: Install Native Dependencies + run: | + sudo dpkg --add-architecture i386 + sudo apt-get update + sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb libgcc-s1:i386 libgdiplus + - name: Setup Node.JS uses: actions/setup-node@v4 with: diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index b1b83c4a496..ec00f8cf205 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1679,6 +1679,7 @@ enum CheckMode { Pending, Started, + Rerun, Cancelled, Success, Failure, @@ -1715,6 +1716,18 @@ static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSh StartedAt = DateTimeOffset.UtcNow, }); + break; + case CheckMode.Rerun: + var prChecks3 = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); + var theCheckWeWant3 = prChecks3.CheckRuns.First(x => x.App.Id == AppId); + if(theCheckWeWant3.Status != CheckStatus.InProgress) + { + await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant3.Id, new CheckRunUpdate + { + Status = CheckStatus.InProgress, + }); + } + break; case CheckMode.Cancelled: case CheckMode.Failure: From 7920871c369452f570040987b419f210b7b134da Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 22:57:59 -0400 Subject: [PATCH 073/111] Fix security dispatch parameters --- .github/workflows/ci-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 2a49ee9d24f..5feff359c74 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -100,7 +100,7 @@ jobs: workflow-inputs: | { "pull_request_number": "${{ github.event.pull_request.number }}", - "pull_request_head": "${{ github.event.pull_request.head.sha }}" + "pull_request_head_sha": "${{ github.event.pull_request.head.sha }}" } - name: Set CI Check Run (Pending) From 47cf7123cf8b8a03baf8d5ebd96ed7e7dddfb45e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 22:58:15 -0400 Subject: [PATCH 074/111] Remove all unnecessary explicit restores --- .github/workflows/ci-pipeline.yml | 29 -------------------- .github/workflows/ci-security.yml | 3 -- .github/workflows/stable-merge.yml | 5 ---- .github/workflows/update-ss13-org-mirror.yml | 8 ------ build/package/deb/debian/rules | 1 - 5 files changed, 46 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index ae3b13a9ebc..8f423c97fc5 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -94,9 +94,6 @@ jobs: await delay(5000); } - - name: Restore - run: dotnet restore - - name: Build ReleaseNotes run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj @@ -350,11 +347,6 @@ jobs: git checkout ${{ matrix.committish }} git submodule update --init --recursive - - name: Restore OpenDream - run: | - cd $HOME/OpenDream - dotnet restore - - name: Build OpenDream run: | cd $HOME/OpenDream/OpenDreamPackageTool @@ -632,9 +624,6 @@ jobs: await delay(5000); } - - name: Restore - run: dotnet restore - - name: Enable Corepack run: corepack enable @@ -722,9 +711,6 @@ jobs: await delay(5000); } - - name: Restore - run: dotnet restore - - name: Enable Corepack run: corepack enable @@ -889,9 +875,6 @@ jobs: await delay(5000); } - - name: Restore - run: dotnet restore - - name: Enable Corepack run: corepack enable @@ -1146,9 +1129,6 @@ jobs: await delay(5000); } - - name: Restore - run: dotnet restore - - name: Enable Corepack run: corepack enable @@ -1750,9 +1730,6 @@ jobs: - name: Validate winget Manifest run: winget validate --manifest build/package/winget/manifest - - name: Restore - run: dotnet restore - - name: Enable Corepack run: corepack enable @@ -2085,9 +2062,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml @@ -2167,9 +2141,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Restore - run: dotnet restore - - name: Restore Wix dotnet Tool run: | cd build/package/winget diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 5feff359c74..9729a2120db 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -76,9 +76,6 @@ jobs: with: ref: refs/pull/${{ github.event.pull_request.number }}/merge - - name: Restore - run: dotnet restore - - name: Build ReleaseNotes run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj diff --git a/.github/workflows/stable-merge.yml b/.github/workflows/stable-merge.yml index 2a9a6eb7260..509c9e85c56 100644 --- a/.github/workflows/stable-merge.yml +++ b/.github/workflows/stable-merge.yml @@ -26,11 +26,6 @@ jobs: with: path: temp_workspace - - name: Restore - run: | - cd temp_workspace - dotnet restore - - name: Build ReleaseNotes run: | cd temp_workspace diff --git a/.github/workflows/update-ss13-org-mirror.yml b/.github/workflows/update-ss13-org-mirror.yml index cbac03119b1..f6562220f7b 100644 --- a/.github/workflows/update-ss13-org-mirror.yml +++ b/.github/workflows/update-ss13-org-mirror.yml @@ -32,11 +32,6 @@ jobs: with: path: temp_workspace - - name: Restore - run: | - cd temp_workspace - dotnet restore - - name: Build ReleaseNotes run: | cd temp_workspace @@ -58,9 +53,6 @@ jobs: fetch-tags: true token: ${{ env.INSTALLATION_TOKEN }} - - name: Restore - run: dotnet restore - - name: Build ReleaseNotes run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj diff --git a/build/package/deb/debian/rules b/build/package/deb/debian/rules index 7a13e96dd26..cbd846c2da9 100755 --- a/build/package/deb/debian/rules +++ b/build/package/deb/debian/rules @@ -10,7 +10,6 @@ override_dh_auto_clean: dotnet clean -c ReleaseNoWindows override_dh_auto_build: - dotnet restore cd src/Tgstation.Server.Host.Console && dotnet publish -c Release -o ../../artifacts cd src/Tgstation.Server.Host && dotnet publish -c Release -o ../../artifacts/lib/Default rm artifacts/lib/Default/appsettings.yml From 8247edc0955513f94c23d7933d83192b5e5339a5 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 23:04:19 -0400 Subject: [PATCH 075/111] Fix bad command lines --- .github/workflows/ci-pipeline.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 8f423c97fc5..e790c12c5f1 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -104,7 +104,7 @@ jobs: path: ./release_notes_bins/ - name: Set CI Check Run (Started) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} code-scanning: name: Code Scanning @@ -771,7 +771,7 @@ jobs: path: release_notes_bins - name: Update CI Check Run (Rerun) # Do this here because these are the flakiest tests - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} - name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later shell: powershell @@ -1050,7 +1050,7 @@ jobs: path: release_notes_bins - name: Update CI Check Run (Rerun) # Do this here because these are the flakiest tests - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} - name: Disable ptrace_scope run: echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope @@ -1881,15 +1881,15 @@ jobs: - name: Update CI Check Run (Cancelled) if: needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled' - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - name: Update CI Check Run (Failure) if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - name: Update CI Check Run (Success) if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.code-scanning.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success' - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }}{{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - name: Fail Job if Prerequisites Failed if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) From 9c1ee13d70362e9b40fa18140684d2fe75b5ffc9 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 23:04:32 -0400 Subject: [PATCH 076/111] Actually enable check run ID discovery --- .github/workflows/ci-security.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 9729a2120db..f9998cff172 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -94,6 +94,7 @@ jobs: ref: ${{ github.event.pull_request.number }}-merge workflow: ci-pipeline.yml token: ${{ github.token }} + discover: true workflow-inputs: | { "pull_request_number": "${{ github.event.pull_request.number }}", From 6721a2a0529a7038235195dcc65e6ce57c397f91 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 23:11:55 -0400 Subject: [PATCH 077/111] Adjust CI Pipeline to work with discovery --- .github/workflows/ci-pipeline.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index e790c12c5f1..3469d252068 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -17,6 +17,7 @@ # - apt repo: https://github.com/tgstation/tgstation-ppa # - winget: https://github.com/microsoft/winget-pkgs/tree/master/manifests/t/Tgstation/Server name: 'CI Pipeline' +run-name: CI Pipeline [${{ inputs.distinct_id && inputs.distinct_id || github.head_ref }}] on: schedule: @@ -33,6 +34,9 @@ on: pull_request_head_sha: description: 'Pull Request HEAD SHA' required: true + distinct_id: + description: 'Distinct ID' + required: true env: TGS_DOTNET_VERSION: 8 From c1296b1b13e21ff0b7fcb2638977a739a811389e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 23:19:44 -0400 Subject: [PATCH 078/111] Add missing checkout step --- .github/workflows/ci-pipeline.yml | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 3469d252068..209acb024ea 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1850,6 +1850,40 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Checkout (Branch) + uses: actions/checkout@v4 + if: github.event_name == 'push' || github.event_name == 'schedule' + + - name: Checkout (PR Merge) + uses: actions/checkout@v4 + if: github.event_name != 'push' && github.event_name != 'schedule' + with: + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } + - name: Retrieve Latest winget-pkgs PULL_REQUEST_TEMPLATE commit SHA from GitHub API id: get-sha run: | From 5c42a15455b4c7185d1a2aa7d97bc5e595b94ade Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 23:21:12 -0400 Subject: [PATCH 079/111] Fix ref names in run titles --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 209acb024ea..ca3fdb13391 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -17,7 +17,7 @@ # - apt repo: https://github.com/tgstation/tgstation-ppa # - winget: https://github.com/microsoft/winget-pkgs/tree/master/manifests/t/Tgstation/Server name: 'CI Pipeline' -run-name: CI Pipeline [${{ inputs.distinct_id && inputs.distinct_id || github.head_ref }}] +run-name: CI Pipeline [${{ inputs.distinct_id && inputs.distinct_id || github.ref_name }}] on: schedule: From 05410145af3d764590f0e315021fd87f43c6058c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 23:36:11 -0400 Subject: [PATCH 080/111] Add missing checkout for pages build --- .github/workflows/ci-pipeline.yml | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index ca3fdb13391..dcb351d3fa0 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -458,6 +458,40 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + - name: Checkout (Branch) + uses: actions/checkout@v4 + if: github.event_name == 'push' || github.event_name == 'schedule' + + - name: Checkout (PR Merge) + uses: actions/checkout@v4 + if: github.event_name != 'push' && github.event_name != 'schedule' + with: + ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } + - name: gh-pages Clone run: git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox From a6bc0815dc0869589754f0f23d742a600e35afb8 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 23:39:27 -0400 Subject: [PATCH 081/111] Extract code scanning into its own workflow --- .github/workflows/ci-pipeline.yml | 80 ++--------------------------- .github/workflows/code-scanning.yml | 74 ++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 75 deletions(-) create mode 100644 .github/workflows/code-scanning.yml diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index dcb351d3fa0..aa40f5ddd36 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -110,76 +110,6 @@ jobs: - name: Set CI Check Run (Started) run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} - code-scanning: - name: Code Scanning - needs: ci-start-gate - runs-on: ubuntu-latest - permissions: - security-events: write - actions: read - env: - TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt - steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Checkout (Branch) - uses: actions/checkout@v4 - if: github.event_name == 'push' || github.event_name == 'schedule' - - - name: Checkout (PR Merge) - uses: actions/checkout@v4 - if: github.event_name != 'push' && github.event_name != 'schedule' - with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" - - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: csharp - - - name: Setup Telemetry Key File - run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - - - name: Build - run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true - - - name: Delete Telemetry Key File - if: always() - run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:csharp" - dmapi-build: name: Build DMAPI needs: ci-start-gate @@ -1935,7 +1865,7 @@ jobs: ci-completion-gate: name: CI Completion Gate - needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning, efcore-version-match, ci-start-gate ] + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, efcore-version-match, ci-start-gate ] runs-on: ubuntu-latest if: always() && needs.ci-start-gate.result == 'success' steps: @@ -1952,19 +1882,19 @@ jobs: path: release_notes_bins - name: Update CI Check Run (Cancelled) - if: needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled' + if: needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled' run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - name: Update CI Check Run (Failure) - if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) + if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - name: Update CI Check Run (Success) - if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.code-scanning.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success' + if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success' run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - name: Fail Job if Prerequisites Failed - if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.code-scanning.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.code-scanning.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) + if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) run: exit 1 deployment-gate: diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml new file mode 100644 index 00000000000..dd9e51bbeb2 --- /dev/null +++ b/.github/workflows/code-scanning.yml @@ -0,0 +1,74 @@ +name: Code Scanning + +on: + pull_request: + branches: + - dev + - master + +concurrency: + group: "code-scanning-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" + cancel-in-progress: true + +env: + TGS_DOTNET_VERSION: 8 + TGS_DOTNET_QUALITY: ga + +jobs: + code-scanning: + name: Run CodeQL + runs-on: ubuntu-latest + permissions: + security-events: write + actions: read + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt + steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Checkout (Branch) + uses: actions/checkout@v4 + + - name: Read Current SHA + id: get-pr-sha + if: github.event_name != 'push' && github.event_name != 'schedule' + shell: bash + run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - name: Abort if PR Merge SHA has Changed + uses: actions/github-script@v7 + if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + with: + script: | + const delay = ms => new Promise(res => setTimeout(res, ms)); + + github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + while (true) { + core.info('Waiting for workflow to cancel ...'); + await delay(5000); + } + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: csharp + + - name: Setup Telemetry Key File + run: echo "fake_telemetry_key" > ${{ env.TGS_TELEMETRY_KEY_FILE }} + + - name: Build + run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:csharp" From bbb1fca66d7385056d08d613fec2c5d93b5f6083 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 23:40:13 -0400 Subject: [PATCH 082/111] Fix formatting --- .github/workflows/ci-pipeline.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index aa40f5ddd36..aa9c6770b3c 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -72,7 +72,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -170,7 +170,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -247,7 +247,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -309,7 +309,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -396,7 +396,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -491,7 +491,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -566,7 +566,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -653,7 +653,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -817,7 +817,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -1071,7 +1071,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -1180,7 +1180,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -1228,7 +1228,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -1541,7 +1541,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -1664,7 +1664,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha @@ -1822,7 +1822,7 @@ jobs: uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: - ref: "refs/pull/${{ inputs.pull_request_number}}/merge" + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Read Current SHA id: get-pr-sha From 27c8df29883d68d717d217422ec7ef3e283cdc6e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 23:41:08 -0400 Subject: [PATCH 083/111] Add missing push events to code scanning workflow --- .github/workflows/code-scanning.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index dd9e51bbeb2..f1fe0397eef 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -1,6 +1,10 @@ name: Code Scanning on: + push: + branches: + - dev + - master pull_request: branches: - dev From 9ca1d80037835b576a63e1f71a72712cf41d29f4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 16 Aug 2024 23:43:23 -0400 Subject: [PATCH 084/111] Fix check handling command lines for branch builds --- .github/workflows/ci-pipeline.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index aa9c6770b3c..c45546f67c1 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -108,7 +108,7 @@ jobs: path: ./release_notes_bins/ - name: Set CI Check Run (Started) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} dmapi-build: name: Build DMAPI @@ -739,7 +739,7 @@ jobs: path: release_notes_bins - name: Update CI Check Run (Rerun) # Do this here because these are the flakiest tests - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} - name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later shell: powershell @@ -1018,7 +1018,7 @@ jobs: path: release_notes_bins - name: Update CI Check Run (Rerun) # Do this here because these are the flakiest tests - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} - name: Disable ptrace_scope run: echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope @@ -1883,15 +1883,15 @@ jobs: - name: Update CI Check Run (Cancelled) if: needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled' - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - name: Update CI Check Run (Failure) if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - name: Update CI Check Run (Success) if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success' - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - name: Fail Job if Prerequisites Failed if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) From 4dd37e7ef3dfbabb4a51645f87f13240042ed121 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 08:12:25 -0400 Subject: [PATCH 085/111] Fix code coverage uploading on PRs --- .github/workflows/ci-pipeline.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index c45546f67c1..da9edcf9b8b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1488,11 +1488,21 @@ jobs: name: windows-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlite - - name: Upload Coverage to CodeCov + - name: Upload Coverage to CodeCov (Branch) + if: github.event_name == 'push' || github.event_name == 'schedule' + uses: codecov/codecov-action@v3 + with: + directory: ./code_coverage + fail_ci_if_error: true + + - name: Upload Coverage to CodeCov (PR) + if: github.event_name != 'push' && github.event_name != 'schedule' uses: codecov/codecov-action@v3 with: directory: ./code_coverage fail_ci_if_error: true + override_pr: ${{ inputs.pull_request_number }} + override_commit: ${{ github.sha }} build-deb: name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595 From a40b6c397aa4dbe8d9f7ed1f2ebb747c772aaca0 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 08:26:08 -0400 Subject: [PATCH 086/111] Hopefully fix re-run issues --- .../Tgstation.Server.ReleaseNotes/Program.cs | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index ec00f8cf205..cd76b6c3a33 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1699,7 +1699,7 @@ static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSh break; case CheckMode.Started: var prChecks = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); - var theCheckWeWant = prChecks.CheckRuns.FirstOrDefault(x => x.App.Id == AppId); + var theCheckWeWant = prChecks.CheckRuns.FirstOrDefault(x => x.App.Id == AppId && x.Status != CheckStatus.Completed); if (theCheckWeWant != null) { await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate @@ -1720,13 +1720,20 @@ static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSh case CheckMode.Rerun: var prChecks3 = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); var theCheckWeWant3 = prChecks3.CheckRuns.First(x => x.App.Id == AppId); - if(theCheckWeWant3.Status != CheckStatus.InProgress) + if(theCheckWeWant3.Status == CheckStatus.Completed) { - await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant3.Id, new CheckRunUpdate + // need a new check run + await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) { Status = CheckStatus.InProgress, + DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", + StartedAt = DateTimeOffset.UtcNow, }); } + else + { + Console.WriteLine($"Check status is {theCheckWeWant3.Status}"); + } break; case CheckMode.Cancelled: @@ -1741,13 +1748,27 @@ static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSh }; var prChecks2 = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); - var theCheckWeWant2 = prChecks2.CheckRuns.First(x => x.App.Id == AppId); - await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant2.Id, new CheckRunUpdate + var theCheckWeWant2 = prChecks2.CheckRuns.FirstOrDefault(x => x.App.Id == AppId && x.Status != CheckStatus.Completed); + if (theCheckWeWant2 != null) { - Status = CheckStatus.Completed, - CompletedAt = DateTimeOffset.UtcNow, - Conclusion = conclusion, - }); + await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant2.Id, new CheckRunUpdate + { + Status = CheckStatus.Completed, + CompletedAt = DateTimeOffset.UtcNow, + Conclusion = conclusion, + }); + } + else + { + // need a new check run + await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) + { + Status = CheckStatus.Completed, + DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", + CompletedAt = DateTimeOffset.UtcNow, + Conclusion = conclusion, + }); + } break; } From 54da95512be22117600b4292582be2f0b8bb8c2c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 08:33:37 -0400 Subject: [PATCH 087/111] Prevent unnecessary API call --- tools/Tgstation.Server.ReleaseNotes/Program.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index cd76b6c3a33..71f7cdf9229 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1702,11 +1702,14 @@ static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSh var theCheckWeWant = prChecks.CheckRuns.FirstOrDefault(x => x.App.Id == AppId && x.Status != CheckStatus.Completed); if (theCheckWeWant != null) { - await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate + if (theCheckWeWant.Status != CheckStatus.InProgress) { - Status = CheckStatus.InProgress, - StartedAt = DateTimeOffset.UtcNow, - }); + await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate + { + Status = CheckStatus.InProgress, + StartedAt = DateTimeOffset.UtcNow, + }); + } } else await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) From 293c4870a0555873c491df4f433640469a726828 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 09:56:52 -0400 Subject: [PATCH 088/111] Fix more stupid needs --- .github/workflows/ci-pipeline.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index da9edcf9b8b..b31229a492c 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -2418,7 +2418,6 @@ jobs: deploy-docker: name: Deploy TGS (Docker) needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') runs-on: ubuntu-latest steps: - name: Checkout @@ -2442,7 +2441,6 @@ jobs: deploy-ppa: name: Deploy TGS (PPA) needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') runs-on: ubuntu-latest steps: - name: Checkout @@ -2461,7 +2459,6 @@ jobs: deploy-winget: name: Deploy TGS (winget) needs: deploy-tgs - if: (!(cancelled() || failure()) && needs.deploy-tgs.result == 'success') runs-on: windows-latest steps: - name: Setup dotnet From 0359535de90ddccd6cb555b81731f9ff71f09f79 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 10:00:02 -0400 Subject: [PATCH 089/111] Fix failure completion gate conditions --- .github/workflows/ci-pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b31229a492c..94ae54e0347 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1895,8 +1895,8 @@ jobs: if: needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled' run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - - name: Update CI Check Run (Failure) - if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) + - name: Update CI Check Run (Failure) # This is just !(cancelled || success) because failure can happen in non-needs and not trigger on any of them + if: (!((needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') || (needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success'))) run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - name: Update CI Check Run (Success) @@ -1904,7 +1904,7 @@ jobs: run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - name: Fail Job if Prerequisites Failed - if: (!(needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') && (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.pages-build.result == 'failure')) + if: (!((needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') || (needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success'))) run: exit 1 deployment-gate: From 4e2a4b024ebaae5418a927c37530578f989f5001 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 10:43:36 -0400 Subject: [PATCH 090/111] So re-runs are new run IDs, handle that --- .../Tgstation.Server.ReleaseNotes/Program.cs | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index 71f7cdf9229..a23479de574 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1689,18 +1689,25 @@ static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSh { await GenerateAppCredentials(gitHubClient, pemBase64, false); + const string CheckName = "CI Pipeline"; + var detailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}"; + + if (mode == CheckMode.Pending) + { + await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) + { + DetailsUrl = detailsUrl + }); + + return 0; + } + + var prChecks = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); + var theCheckWeWant = prChecks.CheckRuns.FirstOrDefault(x => x.Name == CheckName && x.DetailsUrl == detailsUrl); switch (mode) { - case CheckMode.Pending: - await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) - { - DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", - }); - break; case CheckMode.Started: - var prChecks = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); - var theCheckWeWant = prChecks.CheckRuns.FirstOrDefault(x => x.App.Id == AppId && x.Status != CheckStatus.Completed); - if (theCheckWeWant != null) + if (theCheckWeWant != null && theCheckWeWant.Status != CheckStatus.Completed) { if (theCheckWeWant.Status != CheckStatus.InProgress) { @@ -1714,16 +1721,14 @@ static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSh else await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) { - DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", Status = CheckStatus.InProgress, StartedAt = DateTimeOffset.UtcNow, + DetailsUrl = detailsUrl, }); break; case CheckMode.Rerun: - var prChecks3 = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); - var theCheckWeWant3 = prChecks3.CheckRuns.First(x => x.App.Id == AppId); - if(theCheckWeWant3.Status == CheckStatus.Completed) + if(theCheckWeWant != null && theCheckWeWant.Status == CheckStatus.Completed) { // need a new check run await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) @@ -1733,9 +1738,13 @@ static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSh StartedAt = DateTimeOffset.UtcNow, }); } - else + else if (theCheckWeWant.Status != CheckStatus.InProgress) { - Console.WriteLine($"Check status is {theCheckWeWant3.Status}"); + await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate + { + Status = CheckStatus.InProgress, + StartedAt = DateTimeOffset.UtcNow, + }); } break; @@ -1750,11 +1759,9 @@ static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSh _ => throw new InvalidOperationException("Impossible"), }; - var prChecks2 = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); - var theCheckWeWant2 = prChecks2.CheckRuns.FirstOrDefault(x => x.App.Id == AppId && x.Status != CheckStatus.Completed); - if (theCheckWeWant2 != null) + if (theCheckWeWant != null && theCheckWeWant.Status != CheckStatus.Completed) { - await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant2.Id, new CheckRunUpdate + await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate { Status = CheckStatus.Completed, CompletedAt = DateTimeOffset.UtcNow, From 70a137615599098eaaa29bacc6ac51aeede9aa65 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 10:53:13 -0400 Subject: [PATCH 091/111] Hopefully fix run completion conditionals --- .github/workflows/ci-pipeline.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 94ae54e0347..ac588c2d808 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1892,19 +1892,19 @@ jobs: path: release_notes_bins - name: Update CI Check Run (Cancelled) - if: needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled' + if: cancelled() run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - name: Update CI Check Run (Failure) # This is just !(cancelled || success) because failure can happen in non-needs and not trigger on any of them - if: (!((needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') || (needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success'))) + if: (!(cancelled() || (needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success'))) run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - - name: Update CI Check Run (Success) - if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success' + - name: Update CI Check Run (Success) # Only need to check direct dependencies + if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.ci-start-gate.result == 'success' run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - name: Fail Job if Prerequisites Failed - if: (!((needs.pages-build.result == 'cancelled' || needs.docker-build.result == 'cancelled' || needs.build-deb.result == 'cancelled' || needs.build-msi.result == 'cancelled' || needs.validate-openapi-spec.result == 'cancelled' || needs.upload-code-coverage.result == 'cancelled' || needs.check-winget-pr-template.result == 'cancelled' || needs.efcore-version-match.result == 'cancelled' || needs.pages-build.result == 'cancelled') || (needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.pages-build.result == 'success'))) + if: (!(cancelled() || (needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.ci-start-gate.result == 'success'))) run: exit 1 deployment-gate: From 31669e18d619e40d59d6c9347f5730ad4eac87c8 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 11:05:05 -0400 Subject: [PATCH 092/111] Truly fix run completion conditionals --- .github/workflows/ci-pipeline.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index ac588c2d808..5616b63624c 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -710,7 +710,7 @@ jobs: name: windows-unit-test-coverage-${{ matrix.configuration }} path: ./TestResults/ - windows-integration-test: + windows-integration-tests: name: Windows Live Tests needs: [dmapi-build, opendream-build] strategy: @@ -1166,7 +1166,7 @@ jobs: validate-openapi-spec: name: OpenAPI Spec Validation - needs: windows-integration-test + needs: windows-integration-tests runs-on: ubuntu-latest steps: - name: Install IBM OpenAPI Validator @@ -1217,7 +1217,7 @@ jobs: upload-code-coverage: name: Upload Code Coverage - needs: [linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-test] + needs: [ linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-tests ] runs-on: ubuntu-latest steps: - name: Checkout (Branch) @@ -1875,7 +1875,7 @@ jobs: ci-completion-gate: name: CI Completion Gate - needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, efcore-version-match, ci-start-gate ] + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, efcore-version-match, ci-start-gate, dmapi-build, opendream-build, windows-unit-tests, linux-unit-tests, windows-integration-tests, linux-integration-tests ] runs-on: ubuntu-latest if: always() && needs.ci-start-gate.result == 'success' steps: @@ -1891,12 +1891,12 @@ jobs: name: release_notes_bins path: release_notes_bins - - name: Update CI Check Run (Cancelled) - if: cancelled() + - name: Update CI Check Run (Cancelled) # !(success() || failure()) + if: (!((needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.ci-start-gate.result == 'success') || (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.ci-start-gate.result == 'failure' || needs.dmapi-build.result == 'failure' || needs.opendream-build.result == 'failure' || needs.windows-unit-tests.result == 'failure' || needs.linux-unit-tests.result == 'failure' || needs.windows-integration-tests.result == 'failure' || needs.linux-integration-tests.result == 'failure'))) run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - - name: Update CI Check Run (Failure) # This is just !(cancelled || success) because failure can happen in non-needs and not trigger on any of them - if: (!(cancelled() || (needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success'))) + - name: Update CI Check Run (Failure) + if: needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.ci-start-gate.result == 'failure' || needs.dmapi-build.result == 'failure' || needs.opendream-build.result == 'failure' || needs.windows-unit-tests.result == 'failure' || needs.linux-unit-tests.result == 'failure' || needs.windows-integration-tests.result == 'failure' || needs.linux-integration-tests.result == 'failure' run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - name: Update CI Check Run (Success) # Only need to check direct dependencies @@ -1904,7 +1904,7 @@ jobs: run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - name: Fail Job if Prerequisites Failed - if: (!(cancelled() || (needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.ci-start-gate.result == 'success'))) + if: needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.ci-start-gate.result == 'failure' || needs.dmapi-build.result == 'failure' || needs.opendream-build.result == 'failure' || needs.windows-unit-tests.result == 'failure' || needs.linux-unit-tests.result == 'failure' || needs.windows-integration-tests.result == 'failure' || needs.linux-integration-tests.result == 'failure' run: exit 1 deployment-gate: From 60c1e182f4dd1363e0cf3ee9c501d0abd89c40cf Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 11:10:26 -0400 Subject: [PATCH 093/111] Fix telemetry key removal steps --- .github/workflows/ci-pipeline.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 5616b63624c..f72e66d5167 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -526,7 +526,7 @@ jobs: - name: Delete Telemetry Key File if: always() - run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} linux-unit-tests: name: Linux Tests @@ -603,7 +603,7 @@ jobs: - name: Delete Telemetry Key File if: always() - run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Cache BYOND .zips uses: actions/cache@v4 @@ -692,7 +692,7 @@ jobs: - name: Delete Telemetry Key File shell: bash if: always() - run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Cache BYOND .zips uses: actions/cache@v4 @@ -856,7 +856,7 @@ jobs: - name: Delete Telemetry Key File shell: bash if: always() - run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Cache BYOND .zips uses: actions/cache@v4 @@ -1108,7 +1108,7 @@ jobs: - name: Delete Telemetry Key File if: always() - run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Cache BYOND .zips uses: actions/cache@v4 @@ -1606,7 +1606,7 @@ jobs: - name: Delete Telemetry Key File if: always() - run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Test Install run: | @@ -1721,7 +1721,7 @@ jobs: - name: Delete Telemetry Key File shell: bash if: always() - run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj @@ -2171,7 +2171,7 @@ jobs: - name: Delete Telemetry Key File shell: bash if: always() - run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }} + run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj From 61f0d9082950476c5c08a2d010cb64ccc75bbebc Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 11:42:50 -0400 Subject: [PATCH 094/111] CI reworking again Switch to reusable workflows --- .github/workflows/ci-pipeline.yml | 460 ++---------------- .github/workflows/ci-security.yml | 61 +-- .github/workflows/code-scanning.yml | 78 --- .../Tgstation.Server.ReleaseNotes/Program.cs | 123 ----- 4 files changed, 55 insertions(+), 667 deletions(-) delete mode 100644 .github/workflows/code-scanning.yml diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index f72e66d5167..9e8579a23db 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -17,7 +17,6 @@ # - apt repo: https://github.com/tgstation/tgstation-ppa # - winget: https://github.com/microsoft/winget-pkgs/tree/master/manifests/t/Tgstation/Server name: 'CI Pipeline' -run-name: CI Pipeline [${{ inputs.distinct_id && inputs.distinct_id || github.ref_name }}] on: schedule: @@ -26,17 +25,16 @@ on: branches: - dev - master - workflow_dispatch: + workflow_call: inputs: pull_request_number: description: 'Pull Request Number' required: true + type: string pull_request_head_sha: description: 'Pull Request HEAD SHA' required: true - distinct_id: - description: 'Distinct ID' - required: true + type: string env: TGS_DOTNET_VERSION: 8 @@ -54,8 +52,8 @@ concurrency: cancel-in-progress: true jobs: - ci-start-gate: - name: CI Start Gate + build-releasenotes: + name: Build ReleaseNotes for Other Jobs runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -74,45 +72,57 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + - name: Build ReleaseNotes + run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Store ReleaseNotes Binaries + uses: actions/upload-artifact@v4 + with: + name: release_notes_bins + path: ./release_notes_bins/ + + code-scanning: + name: Run CodeQL + runs-on: ubuntu-latest + permissions: + security-events: write + actions: read + env: + TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt + steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Checkout (Branch) + uses: actions/checkout@v4 + - name: Read Current SHA id: get-pr-sha if: github.event_name != 'push' && github.event_name != 'schedule' shell: bash run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); + languages: csharp - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } + - name: Setup Telemetry Key File + run: echo "fake_telemetry_key" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - - name: Build ReleaseNotes - run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Build + run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true - - name: Store ReleaseNotes Binaries - uses: actions/upload-artifact@v4 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 with: - name: release_notes_bins - path: ./release_notes_bins/ - - - name: Set CI Check Run (Started) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Started ${{ github.run_id }} + category: "/language:csharp" dmapi-build: name: Build DMAPI - needs: ci-start-gate strategy: fail-fast: false matrix: @@ -178,23 +188,6 @@ jobs: shell: bash run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - name: Build DMAPI Test Project run: | @@ -218,7 +211,6 @@ jobs: opendream-build: name: Build DMAPI (OpenDream) - needs: ci-start-gate strategy: fail-fast: false matrix: @@ -249,30 +241,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Checkout OpenDream run: | cd $HOME @@ -298,7 +266,6 @@ jobs: efcore-version-match: name: Check Nuget Versions Match Tools - needs: ci-start-gate runs-on: ubuntu-latest steps: - name: Checkout (Branch) @@ -311,30 +278,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Retrieve dotnet-ef Tool Version id: dotnet-ef-tool run: echo "version=$(cat src/Tgstation.Server.Host/.config/dotnet-tools.json | jq -r '.tools."dotnet-ef".version')" >> $GITHUB_OUTPUT @@ -379,7 +322,7 @@ jobs: pages-build: name: Build gh-pages - needs: ci-start-gate + needs: build-releasenotes runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -398,30 +341,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: gh-pages Clone run: git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox @@ -478,7 +397,6 @@ jobs: docker-build: name: Build Docker Image - needs: ci-start-gate runs-on: ubuntu-latest env: TGS_TELEMETRY_KEY_FILE: tgs_telemetry_key.txt @@ -493,30 +411,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Setup Telemetry Key File shell: bash run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }} @@ -530,7 +424,6 @@ jobs: linux-unit-tests: name: Linux Tests - needs: ci-start-gate strategy: fail-fast: false matrix: @@ -568,30 +461,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Enable Corepack run: corepack enable @@ -623,7 +492,6 @@ jobs: windows-unit-tests: name: Windows Tests - needs: ci-start-gate strategy: fail-fast: false matrix: @@ -655,30 +523,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Enable Corepack run: corepack enable @@ -712,7 +556,7 @@ jobs: windows-integration-tests: name: Windows Live Tests - needs: [dmapi-build, opendream-build] + needs: [ dmapi-build, opendream-build ] strategy: fail-fast: false matrix: @@ -732,15 +576,6 @@ jobs: ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Retrieve ReleaseNotes Binaries - uses: actions/download-artifact@v4 - with: - name: release_notes_bins - path: release_notes_bins - - - name: Update CI Check Run (Rerun) # Do this here because these are the flakiest tests - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} - - name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later shell: powershell if: ${{ matrix.database-type == 'SqlServer' }} @@ -819,30 +654,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Enable Corepack run: corepack enable @@ -1011,15 +822,6 @@ jobs: ${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Retrieve ReleaseNotes Binaries - uses: actions/download-artifact@v4 - with: - name: release_notes_bins - path: release_notes_bins - - - name: Update CI Check Run (Rerun) # Do this here because these are the flakiest tests - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Rerun ${{ github.run_id }} - - name: Disable ptrace_scope run: echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope @@ -1073,30 +875,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Enable Corepack run: corepack enable @@ -1182,30 +960,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Retrieve OpenAPI Spec uses: actions/download-artifact@v4 with: @@ -1230,30 +984,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Retrieve Linux Unit Test Coverage (Debug) uses: actions/download-artifact@v4 with: @@ -1506,7 +1236,6 @@ jobs: build-deb: name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595 - needs: ci-start-gate runs-on: ubuntu-latest env: TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt @@ -1553,30 +1282,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Grab Most Recent Changelog run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml @@ -1645,7 +1350,6 @@ jobs: build-msi: name: Build Windows Installer .exe - needs: ci-start-gate runs-on: windows-latest env: TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt @@ -1676,30 +1380,6 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Restore Wix dotnet Tool run: | cd build/package/winget @@ -1815,7 +1495,7 @@ jobs: check-winget-pr-template: name: Check winget-pkgs Pull Request Template is up to date - needs: ci-start-gate + needs: build-releasenotes runs-on: ubuntu-latest steps: - name: Setup dotnet @@ -1840,24 +1520,6 @@ jobs: shell: bash run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - name: Retrieve Latest winget-pkgs PULL_REQUEST_TEMPLATE commit SHA from GitHub API id: get-sha run: | @@ -1875,37 +1537,11 @@ jobs: ci-completion-gate: name: CI Completion Gate - needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, efcore-version-match, ci-start-gate, dmapi-build, opendream-build, windows-unit-tests, linux-unit-tests, windows-integration-tests, linux-integration-tests ] + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, efcore-version-match, code-scanning ] runs-on: ubuntu-latest - if: always() && needs.ci-start-gate.result == 'success' steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Retrieve ReleaseNotes Binaries - uses: actions/download-artifact@v4 - with: - name: release_notes_bins - path: release_notes_bins - - - name: Update CI Check Run (Cancelled) # !(success() || failure()) - if: (!((needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.ci-start-gate.result == 'success') || (needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.ci-start-gate.result == 'failure' || needs.dmapi-build.result == 'failure' || needs.opendream-build.result == 'failure' || needs.windows-unit-tests.result == 'failure' || needs.linux-unit-tests.result == 'failure' || needs.windows-integration-tests.result == 'failure' || needs.linux-integration-tests.result == 'failure'))) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Cancelled ${{ github.run_id }} - - - name: Update CI Check Run (Failure) - if: needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.ci-start-gate.result == 'failure' || needs.dmapi-build.result == 'failure' || needs.opendream-build.result == 'failure' || needs.windows-unit-tests.result == 'failure' || needs.linux-unit-tests.result == 'failure' || needs.windows-integration-tests.result == 'failure' || needs.linux-integration-tests.result == 'failure' - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Failure ${{ github.run_id }} - - - name: Update CI Check Run (Success) # Only need to check direct dependencies - if: needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.efcore-version-match.result == 'success' && needs.ci-start-gate.result == 'success' - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ inputs.pull_request_head_sha || github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Success ${{ github.run_id }} - - - name: Fail Job if Prerequisites Failed - if: needs.pages-build.result == 'failure' || needs.docker-build.result == 'failure' || needs.build-deb.result == 'failure' || needs.build-msi.result == 'failure' || needs.validate-openapi-spec.result == 'failure' || needs.upload-code-coverage.result == 'failure' || needs.check-winget-pr-template.result == 'failure' || needs.efcore-version-match.result == 'failure' || needs.ci-start-gate.result == 'failure' || needs.dmapi-build.result == 'failure' || needs.opendream-build.result == 'failure' || needs.windows-unit-tests.result == 'failure' || needs.linux-unit-tests.result == 'failure' || needs.windows-integration-tests.result == 'failure' || needs.linux-integration-tests.result == 'failure' - run: exit 1 + - name: Mandatory Empty Step + run: exit 0 deployment-gate: name: Deployment Start Gate diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index f9998cff172..14b60544bad 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -6,9 +6,6 @@ on: branches: - dev - master -env: - TGS_DOTNET_VERSION: 8 - TGS_DOTNET_QUALITY: ga concurrency: group: "ci-security-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" @@ -26,7 +23,7 @@ jobs: if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 uses: thollander/actions-comment-pull-request@1d3973dc4b8e1399c0620d3f2b1aa5e795465308 with: - message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Note that any changes to ci-security.yml will not be reflected in the run. + message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Note that any changes to ci-security.yml will not be reflected in the run and the ci-pipeline.yml at the HEAD of the pull request will be used. - name: Comment on dependabot PR if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id == 49699333 @@ -57,53 +54,9 @@ jobs: if: (github.event.action == 'synchronize' || github.event.action == 'reopened') || ((github.event.action == 'opened' || github.event.action == 'labeled') && !contains(github.event.pull_request.labels.*.name, 'CI Cleared')) run: exit 1 - ci-dispatch: - name: Start CI Pipeline - runs-on: ubuntu-latest - needs: security-checkpoint - permissions: - actions: write - contents: write - steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Checkout - uses: actions/checkout@v4 - with: - ref: refs/pull/${{ github.event.pull_request.number }}/merge - - - name: Build ReleaseNotes - run: dotnet publish -c Release -p:TGS_HOST_NO_WEBPANEL=true -o release_notes_bins tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - - - name: Generate Temporary Branch to Reference Merge - run: | - git checkout -b ${{ github.event.pull_request.number }}-merge - git push -f -u origin ${{ github.event.pull_request.number }}-merge - - - name: Send Workflow Dispatch - uses: lasith-kg/dispatch-workflow@5623bf13f09bbbbdb549ec692b070307f39b66ac #v2.0.0 + setup_node@v4 - id: dispatch - with: - dispatch-method: workflow_dispatch - owner: ${{ github.repository_owner }} - repo: ${{ github.event.pull_request.base.repo.name }} - ref: ${{ github.event.pull_request.number }}-merge - workflow: ci-pipeline.yml - token: ${{ github.token }} - discover: true - workflow-inputs: | - { - "pull_request_number": "${{ github.event.pull_request.number }}", - "pull_request_head_sha": "${{ github.event.pull_request.head.sha }}" - } - - - name: Set CI Check Run (Pending) - run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes.dll --ci-check ${{ github.event.pull_request.head.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }} Pending ${{ steps.dispatch.outputs.run-id }} - - - name: Delete Temporary Branch - if: always() - run: git push -d origin ${{ github.event.pull_request.number }}-merge + ci-pipline-workflow-call: + uses: ./.github/workflows/ci-pipeline.yml@${{ github.event.pull_request.head.sha }} + secrets: inherit + with: + pull_request_number: ${{ github.event.pull_request.number }} + pull_request_head_sha: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml deleted file mode 100644 index f1fe0397eef..00000000000 --- a/.github/workflows/code-scanning.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Code Scanning - -on: - push: - branches: - - dev - - master - pull_request: - branches: - - dev - - master - -concurrency: - group: "code-scanning-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" - cancel-in-progress: true - -env: - TGS_DOTNET_VERSION: 8 - TGS_DOTNET_QUALITY: ga - -jobs: - code-scanning: - name: Run CodeQL - runs-on: ubuntu-latest - permissions: - security-events: write - actions: read - env: - TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt - steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Checkout (Branch) - uses: actions/checkout@v4 - - - name: Read Current SHA - id: get-pr-sha - if: github.event_name != 'push' && github.event_name != 'schedule' - shell: bash - run: echo "head_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Abort if PR Merge SHA has Changed - uses: actions/github-script@v7 - if: github.event_name != 'push' && github.event_name != 'schedule' && steps.get-pr-sha.outputs.head_sha != github.sha - with: - script: | - const delay = ms => new Promise(res => setTimeout(res, ms)); - - github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId - }); - - while (true) { - core.info('Waiting for workflow to cancel ...'); - await delay(5000); - } - - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: csharp - - - name: Setup Telemetry Key File - run: echo "fake_telemetry_key" > ${{ env.TGS_TELEMETRY_KEY_FILE }} - - - name: Build - run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:csharp" diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index a23479de574..46c887fd5f6 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -59,7 +59,6 @@ static async Task Main(string[] args) var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase); - var ciCheck = versionString.Equals("--ci-check", StringComparison.OrdinalIgnoreCase); var genToken = versionString.Equals("--token-output-file", StringComparison.OrdinalIgnoreCase); if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) @@ -68,7 +67,6 @@ static async Task Main(string[] args) && !shaCheck && !fullNotes && !nuget - && !ciCheck && !genToken) { Console.WriteLine("Invalid version: " + versionString); @@ -126,17 +124,6 @@ static async Task Main(string[] args) return await EnsureRelease(client); } - if (ciCheck) - { - if (args.Length < 5) - { - Console.WriteLine("Missing check parameters!"); - return 4543; - } - - return await CICheck(client, args[1], args[2], Enum.Parse(args[3]), Int64.Parse(args[4])); - } - if (genToken) { if (args.Length < 3) @@ -1675,116 +1662,6 @@ static async ValueTask GenerateAppCredentials(GitHubClient gitHubClient, string gitHubClient.Credentials = new Credentials(installToken.Token); } - enum CheckMode - { - Pending, - Started, - Rerun, - Cancelled, - Success, - Failure, - } - - static async ValueTask CICheck(GitHubClient gitHubClient, string ciTargetSha, string pemBase64, CheckMode mode, long runID) - { - await GenerateAppCredentials(gitHubClient, pemBase64, false); - - const string CheckName = "CI Pipeline"; - var detailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}"; - - if (mode == CheckMode.Pending) - { - await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) - { - DetailsUrl = detailsUrl - }); - - return 0; - } - - var prChecks = await gitHubClient.Check.Run.GetAllForReference(RepoOwner, RepoName, ciTargetSha); - var theCheckWeWant = prChecks.CheckRuns.FirstOrDefault(x => x.Name == CheckName && x.DetailsUrl == detailsUrl); - switch (mode) - { - case CheckMode.Started: - if (theCheckWeWant != null && theCheckWeWant.Status != CheckStatus.Completed) - { - if (theCheckWeWant.Status != CheckStatus.InProgress) - { - await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate - { - Status = CheckStatus.InProgress, - StartedAt = DateTimeOffset.UtcNow, - }); - } - } - else - await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) - { - Status = CheckStatus.InProgress, - StartedAt = DateTimeOffset.UtcNow, - DetailsUrl = detailsUrl, - }); - - break; - case CheckMode.Rerun: - if(theCheckWeWant != null && theCheckWeWant.Status == CheckStatus.Completed) - { - // need a new check run - await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) - { - Status = CheckStatus.InProgress, - DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", - StartedAt = DateTimeOffset.UtcNow, - }); - } - else if (theCheckWeWant.Status != CheckStatus.InProgress) - { - await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate - { - Status = CheckStatus.InProgress, - StartedAt = DateTimeOffset.UtcNow, - }); - } - - break; - case CheckMode.Cancelled: - case CheckMode.Failure: - case CheckMode.Success: - var conclusion = mode switch - { - CheckMode.Cancelled => CheckConclusion.Cancelled, - CheckMode.Failure => CheckConclusion.Failure, - CheckMode.Success => CheckConclusion.Success, - _ => throw new InvalidOperationException("Impossible"), - }; - - if (theCheckWeWant != null && theCheckWeWant.Status != CheckStatus.Completed) - { - await gitHubClient.Check.Run.Update(RepoOwner, RepoName, theCheckWeWant.Id, new CheckRunUpdate - { - Status = CheckStatus.Completed, - CompletedAt = DateTimeOffset.UtcNow, - Conclusion = conclusion, - }); - } - else - { - // need a new check run - await gitHubClient.Check.Run.Create(RepoOwner, RepoName, new NewCheckRun("CI Pipeline", ciTargetSha) - { - Status = CheckStatus.Completed, - DetailsUrl = $"https://github.com/{RepoOwner}/{RepoName}/actions/runs/{runID}", - CompletedAt = DateTimeOffset.UtcNow, - Conclusion = conclusion, - }); - } - break; - } - - return 0; - } - static void DebugAssert(bool condition, string message = null) { // This exists because one of the fucking asserts evaluates an enumerable or something and it was getting optimized out in release From 4262e1dee5e26b3c2385cb6c67af71c4b3aef265 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 11:57:39 -0400 Subject: [PATCH 095/111] Maybe GitHub doesn't allow specifying a ref on reusable workflows --- .github/workflows/ci-security.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 14b60544bad..c67a4ca9219 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -1,6 +1,10 @@ name: 'CI Security' on: + pull_request: + branches: + - dev + - master pull_request_target: types: [ opened, reopened, labeled, synchronize ] branches: @@ -14,10 +18,10 @@ concurrency: jobs: security-checkpoint: name: Check CI Clearance + if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' runs-on: ubuntu-latest permissions: pull-requests: write - if: (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' steps: - name: Comment on new Fork PR if: github.event.action == 'opened' && !contains(github.event.pull_request.labels.*.name, 'CI Cleared') && github.event.pull_request.user.id != 49699333 @@ -55,7 +59,9 @@ jobs: run: exit 1 ci-pipline-workflow-call: - uses: ./.github/workflows/ci-pipeline.yml@${{ github.event.pull_request.head.sha }} + needs: security-checkpoint + if: github.event_name != 'pull_request_target' || needs.security-checkpoint.result == 'success' + uses: ./.github/workflows/ci-pipeline.yml secrets: inherit with: pull_request_number: ${{ github.event.pull_request.number }} From 2ed4f8f0b3fc18207feb03078c92daec2fab92fd Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 12:03:54 -0400 Subject: [PATCH 096/111] Fix conditional for CI Pipeline call --- .github/workflows/ci-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index c67a4ca9219..84d5dd44575 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -60,7 +60,7 @@ jobs: ci-pipline-workflow-call: needs: security-checkpoint - if: github.event_name != 'pull_request_target' || needs.security-checkpoint.result == 'success' + if: (!(cancelled() || failure()) && (github.event_name != 'pull_request_target' || needs.security-checkpoint.result == 'success')) uses: ./.github/workflows/ci-pipeline.yml secrets: inherit with: From 6b79bd00f040e719e3f5436b3c6e0daf0aaaa3e4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 12:04:03 -0400 Subject: [PATCH 097/111] Add name to CI Pipeline call --- .github/workflows/ci-security.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 84d5dd44575..608b9c12f1d 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -59,6 +59,7 @@ jobs: run: exit 1 ci-pipline-workflow-call: + name: CI Pipeline needs: security-checkpoint if: (!(cancelled() || failure()) && (github.event_name != 'pull_request_target' || needs.security-checkpoint.result == 'success')) uses: ./.github/workflows/ci-pipeline.yml From 4cdeed0463503066aa2f72180768b198ae57f68e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 12:09:45 -0400 Subject: [PATCH 098/111] Fix CI Pipeline conditional again --- .github/workflows/ci-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 608b9c12f1d..f9285207e66 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -61,7 +61,7 @@ jobs: ci-pipline-workflow-call: name: CI Pipeline needs: security-checkpoint - if: (!(cancelled() || failure()) && (github.event_name != 'pull_request_target' || needs.security-checkpoint.result == 'success')) + if: (!(cancelled() || failure()) && (!(github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333)) || needs.security-checkpoint.result == 'success')) uses: ./.github/workflows/ci-pipeline.yml secrets: inherit with: From 5ad81b657d817e36a5bed017d006cecde641777f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 12:10:45 -0400 Subject: [PATCH 099/111] Remove unused workflow input --- .github/workflows/ci-pipeline.yml | 4 ---- .github/workflows/ci-security.yml | 1 - 2 files changed, 5 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9e8579a23db..b61b1543f83 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -31,10 +31,6 @@ on: description: 'Pull Request Number' required: true type: string - pull_request_head_sha: - description: 'Pull Request HEAD SHA' - required: true - type: string env: TGS_DOTNET_VERSION: 8 diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index f9285207e66..c0cb4ac3b38 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -66,4 +66,3 @@ jobs: secrets: inherit with: pull_request_number: ${{ github.event.pull_request.number }} - pull_request_head_sha: ${{ github.event.pull_request.head.sha }} From a8a193cdcfc3900bd5daf1139e556df4deb6c420 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 12:25:54 -0400 Subject: [PATCH 100/111] Fix CI Pipeline conditional for sure this time --- .github/workflows/ci-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index c0cb4ac3b38..d89d2d3055b 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -61,7 +61,7 @@ jobs: ci-pipline-workflow-call: name: CI Pipeline needs: security-checkpoint - if: (!(cancelled() || failure()) && (!(github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333)) || needs.security-checkpoint.result == 'success')) + if: (!(cancelled() || failure()) && (needs.security-checkpoint.result == 'success' || (github.event_name != 'pull_request_target' && github.event.pull_request.head.repo.id == github.event.pull_request.base.repo.id && github.event.pull_request.user.id != 49699333))) uses: ./.github/workflows/ci-pipeline.yml secrets: inherit with: From 41c95bba7d3aae375d7c26ea33c38a7a9cde285f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 12:34:01 -0400 Subject: [PATCH 101/111] Remove codecov action overrides --- .github/workflows/ci-pipeline.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b61b1543f83..08a7a1cfc53 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1214,21 +1214,11 @@ jobs: name: windows-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlite - - name: Upload Coverage to CodeCov (Branch) - if: github.event_name == 'push' || github.event_name == 'schedule' - uses: codecov/codecov-action@v3 - with: - directory: ./code_coverage - fail_ci_if_error: true - - - name: Upload Coverage to CodeCov (PR) - if: github.event_name != 'push' && github.event_name != 'schedule' + - name: Upload Coverage to CodeCov uses: codecov/codecov-action@v3 with: directory: ./code_coverage fail_ci_if_error: true - override_pr: ${{ inputs.pull_request_number }} - override_commit: ${{ github.sha }} build-deb: name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595 From 5f21e8050f45270f861c8f2442d9efd6b22f3e49 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Aug 2024 16:36:36 +0000 Subject: [PATCH 102/111] Bump Microsoft.EntityFrameworkCore.Sqlite and Microsoft.Bcl.AsyncInterfaces Bumps [Microsoft.EntityFrameworkCore.Sqlite](https://github.com/dotnet/efcore) and Microsoft.Bcl.AsyncInterfaces. These dependencies needed to be updated together. Updates `Microsoft.EntityFrameworkCore.Sqlite` from 8.0.7 to 8.0.8 - [Release notes](https://github.com/dotnet/efcore/releases) - [Commits](https://github.com/dotnet/efcore/compare/v8.0.7...v8.0.8) Updates `Microsoft.Bcl.AsyncInterfaces` from 6.0.0 to 8.0.0 --- updated-dependencies: - dependency-name: Microsoft.EntityFrameworkCore.Sqlite dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: Microsoft.Bcl.AsyncInterfaces dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 1c2937a5b21..8fc7dca66bb 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -112,7 +112,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + From f0bf09665d1749458cfa43265e8c14169e298fee Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 17 Aug 2024 20:27:10 -0400 Subject: [PATCH 103/111] Fix CI Pipeline Badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ea51ba7f332..a646eda7c05 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # tgstation-server -![CI Pipeline](https://github.com/tgstation/tgstation-server/workflows/CI%20Pipeline/badge.svg) [![codecov](https://codecov.io/gh/tgstation/tgstation-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tgstation/tgstation-server) +[![CI Pipeline](https://github.com/tgstation/tgstation-server/actions/workflows/ci-pipeline.yml/badge.svg)](https://github.com/tgstation/tgstation-server/actions/workflows/ci-pipeline.yml) [![codecov](https://codecov.io/gh/tgstation/tgstation-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tgstation/tgstation-server) [![GitHub license](https://img.shields.io/github/license/tgstation/tgstation-server.svg)](LICENSE) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/tgstation/tgstation-server.svg)](http://isitmaintained.com/project/tgstation/tgstation-server "Average time to resolve an issue") [![NuGet version](https://img.shields.io/nuget/v/Tgstation.Server.Api.svg)](https://www.nuget.org/packages/Tgstation.Server.Api) [![NuGet version](https://img.shields.io/nuget/v/Tgstation.Server.Client.svg)](https://www.nuget.org/packages/Tgstation.Server.Client) From df43a072b29fc35c3825dbce230b34880f943056 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 18 Aug 2024 00:00:02 -0400 Subject: [PATCH 104/111] Fix issues with `JobProgressReporter` --- .../Components/Engine/ByondInstallerBase.cs | 2 +- .../Engine/DelegatingEngineInstaller.cs | 2 +- .../Components/Engine/EngineInstallerBase.cs | 2 +- .../Components/Engine/EngineManager.cs | 81 ++++---- .../Components/Engine/IEngineInstaller.cs | 4 +- .../Components/Engine/IEngineManager.cs | 4 +- .../Components/Engine/OpenDreamInstaller.cs | 61 ++++-- .../Components/Repository/IRepository.cs | 8 +- .../Repository/IRepositoryManager.cs | 4 +- .../Components/Repository/Repository.cs | 190 +++++++++++------ .../Repository/RepositoryManager.cs | 6 +- .../Repository/RepositoryUpdateService.cs | 191 ++++++++++-------- .../Controllers/EngineController.cs | 3 +- .../Extensions/FetchOptionsExtensions.cs | 8 +- .../Jobs/JobProgressReporter.cs | 116 ++++++++++- src/Tgstation.Server.Host/Jobs/JobService.cs | 10 +- .../Engine/TestOpenDreamInstaller.cs | 5 +- .../Repository/TestRepositoryManager.cs | 3 +- .../Jobs/TestJobProgressReporter.cs | 66 ++++++ .../Live/Instance/InstanceTest.cs | 3 +- 20 files changed, 524 insertions(+), 245 deletions(-) create mode 100644 tests/Tgstation.Server.Host.Tests/Jobs/TestJobProgressReporter.cs diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index c7c73e3a231..40fbde1a40c 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -146,7 +146,7 @@ await IOManager.DeleteDirectory( } /// - public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? progressReporter, CancellationToken cancellationToken) + public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter progressReporter, CancellationToken cancellationToken) { CheckVersionValidity(version); diff --git a/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs index 91887e73144..c24eee3e8b0 100644 --- a/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs @@ -37,7 +37,7 @@ public IEngineInstallation CreateInstallation(EngineVersion version, string path => DelegateCall(version, installer => installer.CreateInstallation(version, path, installationTask)); /// - public ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken) + public ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken) => DelegateCall(version, installer => installer.DownloadVersion(version, jobProgressReporter, cancellationToken)); /// diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs index 12cf8e657cd..6ca2b940306 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineInstallerBase.cs @@ -52,7 +52,7 @@ protected EngineInstallerBase(IIOManager ioManager, ILogger public abstract ValueTask UpgradeInstallation(EngineVersion version, string path, CancellationToken cancellationToken); /// - public abstract ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken); + public abstract ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken); /// public abstract ValueTask TrustDmbPath(EngineVersion version, string fullDmbPath, CancellationToken cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs index f2a60ae5b9f..970e3ab89d5 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs @@ -118,7 +118,7 @@ public EngineManager(IIOManager ioManager, IEngineInstaller engineInstaller, IEv /// public async ValueTask ChangeVersion( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool allowInstallation, @@ -166,8 +166,11 @@ public async ValueTask UseExecutables(EngineVersion? requ "Acquiring lock on BYOND version {version}...", requiredVersion?.ToString() ?? $"{ActiveVersion} (active)"); var versionToUse = requiredVersion ?? ActiveVersion ?? throw new JobException(ErrorCode.EngineNoVersionsInstalled); + + using var progressReporter = new JobProgressReporter(); + var installLock = await AssertAndLockVersion( - null, + progressReporter, versionToUse, null, requiredVersion != null, @@ -388,7 +391,7 @@ await ValueTaskExtensions.WhenAll( /// /// Ensures a BYOND is installed if it isn't already. /// - /// The optional for the operation. + /// The for the operation. /// The to install. /// Optional custom zip file to use. Will cause a number to be added. /// If this BYOND version is required as part of a locking operation. @@ -396,7 +399,7 @@ await ValueTaskExtensions.WhenAll( /// The for the operation. /// A resulting in the . async ValueTask AssertAndLockVersion( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool neededForLock, @@ -443,8 +446,7 @@ async ValueTask AssertAndLockVersion( { if (installedOrInstalling) { - if (progressReporter != null) - progressReporter.StageName = "Waiting for existing installation job..."; + progressReporter.StageName = "Waiting for existing installation job..."; if (neededForLock && !installation.InstallationTask.IsCompleted) logger.LogWarning("The required engine version ({version}) is not readily available! We will have to wait for it to install.", version); @@ -468,8 +470,7 @@ async ValueTask AssertAndLockVersion( else logger.LogInformation("Requested engine version {version} not currently installed. Doing so now...", version); - if (progressReporter != null) - progressReporter.StageName = "Running event"; + progressReporter.StageName = "Running event"; var versionString = version.ToString(); await eventConsumer.HandleEvent(EventType.EngineInstallStart, new List { versionString }, deploymentPipelineProcesses, cancellationToken); @@ -504,14 +505,14 @@ async ValueTask AssertAndLockVersion( /// /// Installs the files for a given BYOND . /// - /// The optional for the operation. + /// The for the operation. /// The being installed with the number set if appropriate. /// Custom zip file to use. Will cause a number to be added. /// If processes should be launched as part of the deployment pipeline. /// The for the operation. /// A representing the running operation. async ValueTask InstallVersionFiles( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool deploymentPipelineProcesses, @@ -528,14 +529,12 @@ async ValueTask DirectoryCleanup() try { IEngineInstallationData engineInstallationData; + var remainingProgress = 1.0; if (customVersionStream == null) { - if (progressReporter != null) - progressReporter.StageName = "Downloading version"; - - engineInstallationData = await engineInstaller.DownloadVersion(version, progressReporter, cancellationToken); - - progressReporter?.ReportProgress(null); + using var subReporter = progressReporter.CreateSection("Downloading Version", 0.5); + remainingProgress -= 0.5; + engineInstallationData = await engineInstaller.DownloadVersion(version, subReporter, cancellationToken); } else #pragma warning disable CA2000 // Dispose objects before losing scope, false positive @@ -544,33 +543,45 @@ async ValueTask DirectoryCleanup() customVersionStream); #pragma warning restore CA2000 // Dispose objects before losing scope - await using (engineInstallationData) + JobProgressReporter remainingReporter; + try + { + remainingReporter = progressReporter.CreateSection(null, remainingProgress); + } + catch { - if (progressReporter != null) - progressReporter.StageName = "Cleaning target directory"; + await engineInstallationData.DisposeAsync(); + throw; + } - await directoryCleanupTask; + using (remainingReporter) + { + await using (engineInstallationData) + { + remainingReporter.StageName = "Cleaning target directory"; - if (progressReporter != null) - progressReporter.StageName = "Extracting data"; + await directoryCleanupTask; + remainingReporter.ReportProgress(0.1); + remainingReporter.StageName = "Extracting data"; - logger.LogTrace("Extracting engine to {extractPath}...", installFullPath); - await engineInstallationData.ExtractToPath(installFullPath, cancellationToken); - } + logger.LogTrace("Extracting engine to {extractPath}...", installFullPath); + await engineInstallationData.ExtractToPath(installFullPath, cancellationToken); + remainingReporter.ReportProgress(0.3); + } - if (progressReporter != null) - progressReporter.StageName = "Running installation actions"; + remainingReporter.StageName = "Running installation actions"; - await engineInstaller.Install(version, installFullPath, deploymentPipelineProcesses, cancellationToken); + await engineInstaller.Install(version, installFullPath, deploymentPipelineProcesses, cancellationToken); - if (progressReporter != null) - progressReporter.StageName = "Writing version file"; + remainingReporter.ReportProgress(0.9); + remainingReporter.StageName = "Writing version file"; - // make sure to do this last because this is what tells us we have a valid version in the future - await ioManager.WriteAllBytes( - ioManager.ConcatPath(installFullPath, VersionFileName), - Encoding.UTF8.GetBytes(version.ToString()), - cancellationToken); + // make sure to do this last because this is what tells us we have a valid version in the future + await ioManager.WriteAllBytes( + ioManager.ConcatPath(installFullPath, VersionFileName), + Encoding.UTF8.GetBytes(version.ToString()), + cancellationToken); + } } catch (HttpRequestException ex) { diff --git a/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs index a4ad57a4c02..0df793e219c 100644 --- a/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/IEngineInstaller.cs @@ -24,10 +24,10 @@ interface IEngineInstaller /// Download a given engine . /// /// The of the engine to download. - /// The optional for the operation. + /// The for the operation. /// The for the operation. /// A resulting in the for the download. - ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken); + ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken); /// /// Does actions necessary to get an extracted installation working. diff --git a/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs b/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs index 18af1469e6d..7ce4b883db8 100644 --- a/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs +++ b/src/Tgstation.Server.Host/Components/Engine/IEngineManager.cs @@ -28,14 +28,14 @@ public interface IEngineManager : IComponentService, IDisposable /// /// Change the active . /// - /// The optional for the operation. + /// The for the operation. /// The new . /// Optional of a custom BYOND version zip file. /// If an installation should be performed if the is not installed. If and an installation is required an will be thrown. /// The for the operation. /// A representing the running operation. ValueTask ChangeVersion( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, bool allowInstallation, diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index 608d3addb5f..ac4c0964e92 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -133,23 +133,32 @@ public override IEngineInstallation CreateInstallation(EngineVersion version, st } /// - public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter? jobProgressReporter, CancellationToken cancellationToken) + public override async ValueTask DownloadVersion(EngineVersion version, JobProgressReporter jobProgressReporter, CancellationToken cancellationToken) { CheckVersionValidity(version); + ArgumentNullException.ThrowIfNull(jobProgressReporter); // get a lock on a system wide OD repo Logger.LogTrace("Cloning OD repo..."); - var progressSection1 = jobProgressReporter?.CreateSection("Updating OpenDream git repository", 0.5f); - - var repo = await repositoryManager.CloneRepository( - GeneralConfiguration.OpenDreamGitUrl, - null, - null, - null, - progressSection1, - true, - cancellationToken); + var progressSection1 = jobProgressReporter.CreateSection("Updating OpenDream git repository", 0.5f); + IRepository? repo; + try + { + repo = await repositoryManager.CloneRepository( + GeneralConfiguration.OpenDreamGitUrl, + null, + null, + null, + progressSection1, + true, + cancellationToken); + } + catch + { + progressSection1.Dispose(); + throw; + } try { @@ -168,19 +177,23 @@ public override async ValueTask DownloadVersion(EngineV cancellationToken); } - var progressSection2 = jobProgressReporter?.CreateSection("Checking out OpenDream version", 0.5f); + progressSection1.Dispose(); + progressSection1 = null; - var committish = version.SourceSHA - ?? $"{GeneralConfiguration.OpenDreamGitTagPrefix}{version.Version!.Semver()}"; + using (var progressSection2 = jobProgressReporter.CreateSection("Checking out OpenDream version", 0.5f)) + { + var committish = version.SourceSHA + ?? $"{GeneralConfiguration.OpenDreamGitTagPrefix}{version.Version!.Semver()}"; - await repo.CheckoutObject( - committish, - null, - null, - true, - false, - progressSection2, - cancellationToken); + await repo.CheckoutObject( + committish, + null, + null, + true, + false, + progressSection2, + cancellationToken); + } if (!await repo.CommittishIsParent("tgs-min-compat", cancellationToken)) throw new JobException(ErrorCode.OpenDreamTooOld); @@ -192,6 +205,10 @@ await repo.CheckoutObject( repo?.Dispose(); throw; } + finally + { + progressSection1?.Dispose(); + } } /// diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs index e736c4873e1..74417818f19 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs @@ -48,7 +48,7 @@ public interface IRepository : IGitRemoteAdditionalInformation, IDisposable /// The optional password used for fetching from submodule repositories. /// If a submodule update should be attempted after the merge. /// If a hard reset to the target committish should be performed instead of a checkout. - /// The optional to report progress of the operation. + /// The to report progress of the operation. /// The for the operation. /// A representing the running operation. ValueTask CheckoutObject( @@ -57,7 +57,7 @@ ValueTask CheckoutObject( string? password, bool updateSubmodules, bool moveCurrentReference, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, CancellationToken cancellationToken); /// @@ -85,14 +85,14 @@ ValueTask AddTestMerge( /// /// Fetch commits from the origin repository. /// - /// The optional to report progress of the operation. + /// The to report progress of the operation. /// The optional username to fetch from the origin repository. /// The optional password to fetch from the origin repository. /// If any events created should be marked as part of the deployment pipeline. /// The for the operation. /// A representing the running operation. ValueTask FetchOrigin( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, string? username, string? password, bool deploymentPipeline, diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs index 596301611ac..87cbe63d95a 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs @@ -35,7 +35,7 @@ public interface IRepositoryManager : IDisposable /// The optional branch to clone. /// The optional username to clone from . /// The optional password to clone from . - /// The optional for progress of the clone. + /// The for progress of the clone. /// If submodules should be recusively cloned and initialized. /// The for the operation. /// A resulting i the newly cloned , if one already exists. @@ -44,7 +44,7 @@ public interface IRepositoryManager : IDisposable string? initialBranch, string? username, string? password, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, bool recurseSubmodules, CancellationToken cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 6873d429578..9a71bcdcebf 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -232,13 +232,14 @@ await Task.Factory.StartNew( logger.LogTrace("Fetching refspec {refSpec}...", refSpec); var remote = libGitRepo.Network.Remotes.First(); + using var fetchReporter = progressReporter.CreateSection($"Fetch {refSpec}", progressFactor); commands.Fetch( libGitRepo, refSpecList, remote, new FetchOptions().Hydrate( logger, - progressReporter.CreateSection($"Fetch {refSpec}", progressFactor), + fetchReporter, credentialsProvider.GenerateCredentialsHandler(username, password), cancellationToken), logMessage); @@ -267,14 +268,14 @@ await Task.Factory.StartNew( logger.LogTrace("Merging {targetCommitSha} into {currentReference}...", testMergeParameters.TargetCommitSha[..7], Reference); + using var mergeReporter = progressReporter.CreateSection($"Merge {testMergeParameters.TargetCommitSha[..7]}", progressFactor); result = libGitRepo.Merge(testMergeParameters.TargetCommitSha, sig, new MergeOptions { CommitOnSuccess = commitMessage == null, FailOnConflict = false, // Needed to get conflicting files FastForwardStrategy = FastForwardStrategy.NoFastForward, SkipReuc = true, - OnCheckoutProgress = CheckoutProgressHandler( - progressReporter.CreateSection($"Merge {testMergeParameters.TargetCommitSha[..7]}", progressFactor)), + OnCheckoutProgress = CheckoutProgressHandler(mergeReporter), }); } finally @@ -295,7 +296,8 @@ await Task.Factory.StartNew( var revertTo = originalCommit.CanonicalName ?? originalCommit.Tip.Sha; logger.LogDebug("Merge conflict, aborting and reverting to {revertTarget}", revertTo); progressReporter.ReportProgress(0); - RawCheckout(revertTo, false, progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0), cancellationToken); + using var revertReporter = progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0); + RawCheckout(revertTo, false, revertReporter, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); } @@ -343,8 +345,9 @@ await Task.Factory.StartNew( if (updateSubmodules) { + using var progressReporter2 = progressReporter.CreateSection("Update Submodules", progressFactor); await UpdateSubmodules( - progressReporter.CreateSection("Update Submodules", progressFactor), + progressReporter2, username, password, false, @@ -377,7 +380,7 @@ public async ValueTask CheckoutObject( string? password, bool updateSubmodules, bool moveCurrentReference, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(committish); @@ -388,10 +391,11 @@ await Task.Factory.StartNew( () => { libGitRepo.RemoveUntrackedFiles(); + using var progressReporter3 = progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0); RawCheckout( committish, moveCurrentReference, - progressReporter?.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0), + progressReporter3, cancellationToken); }, cancellationToken, @@ -399,17 +403,20 @@ await Task.Factory.StartNew( TaskScheduler.Current); if (updateSubmodules) + { + using var progressReporter2 = progressReporter.CreateSection(null, 1.0 / 3); await UpdateSubmodules( - progressReporter?.CreateSection(null, 1.0 / 3), + progressReporter2, username, password, false, cancellationToken); + } } /// public async ValueTask FetchOrigin( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, string? username, string? password, bool deploymentPipeline, @@ -423,13 +430,14 @@ await Task.Factory.StartNew( var remote = libGitRepo.Network.Remotes.First(); try { + using var subReporter = progressReporter.CreateSection("Fetch Origin", 1.0); var fetchOptions = new FetchOptions { Prune = true, TagFetchMode = TagFetchMode.All, }.Hydrate( logger, - progressReporter?.CreateSection("Fetch Origin", 1.0), + subReporter, credentialsProvider.GenerateCredentialsHandler(username, password), cancellationToken); @@ -471,18 +479,23 @@ public async ValueTask ResetToOrigin( logger.LogTrace("Reset to origin..."); var trackedBranch = libGitRepo.Head.TrackedBranch; await eventConsumer.HandleEvent(EventType.RepoResetOrigin, new List { trackedBranch.FriendlyName, trackedBranch.Tip.Sha }, deploymentPipeline, cancellationToken); - await ResetToSha( - trackedBranch.Tip.Sha, - progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0), - cancellationToken); + + using (var progressReporter2 = progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0)) + await ResetToSha( + trackedBranch.Tip.Sha, + progressReporter2, + cancellationToken); if (updateSubmodules) + { + using var progressReporter3 = progressReporter.CreateSection(null, 1.0 / 3); await UpdateSubmodules( - progressReporter.CreateSection(null, 1.0 / 3), + progressReporter3, username, password, deploymentPipeline, cancellationToken); + } } /// @@ -684,9 +697,10 @@ await eventConsumer.HandleEvent( await Task.Factory.StartNew( () => { + using var resetProgress = progressReporter.CreateSection("Hard reset and remove untracked files", 0.1); libGitRepo.Reset(ResetMode.Hard, libGitRepo.Head.Tip, new CheckoutOptions { - OnCheckoutProgress = CheckoutProgressHandler(progressReporter.CreateSection("Hard reset and remove untracked files", 0.1)), + OnCheckoutProgress = CheckoutProgressHandler(resetProgress), }); cancellationToken.ThrowIfCancellationRequested(); libGitRepo.RemoveUntrackedFiles(); @@ -699,10 +713,11 @@ await Task.Factory.StartNew( var remainingProgressFactor = 0.9; if (!synchronizeTrackedBranch) { + using var progressReporter2 = progressReporter.CreateSection("Push to temporary branch", remainingProgressFactor); await PushHeadToTemporaryBranch( username, password, - progressReporter.CreateSection("Push to temporary branch", remainingProgressFactor), + progressReporter2, cancellationToken); return false; } @@ -722,13 +737,24 @@ await PushHeadToTemporaryBranch( var remote = libGitRepo.Network.Remotes.First(); try { - libGitRepo.Network.Push( - libGitRepo.Head, - GeneratePushOptions( - progressReporter.CreateSection("Push to origin", remainingProgressFactor), - username, - password, - cancellationToken)); + using var pushReporter = progressReporter.CreateSection("Push to origin", remainingProgressFactor); + var (pushOptions, progressReporters) = GeneratePushOptions( + pushReporter, + username, + password, + cancellationToken); + try + { + libGitRepo.Network.Push( + libGitRepo.Head, + pushOptions); + } + finally + { + foreach (var progressReporter in progressReporters) + progressReporter.Dispose(); + } + return true; } catch (NonFastForwardException) @@ -882,9 +908,9 @@ protected override void DisposeImpl() /// /// The committish to checkout. /// If a hard reset should actually be performed. - /// The optional for the operation. + /// The for the operation. /// The for the operation. - void RawCheckout(string committish, bool moveCurrentReference, JobProgressReporter? progressReporter, CancellationToken cancellationToken) + void RawCheckout(string committish, bool moveCurrentReference, JobProgressReporter progressReporter, CancellationToken cancellationToken) { logger.LogTrace("Checkout: {committish}", committish); @@ -893,13 +919,10 @@ void RawCheckout(string committish, bool moveCurrentReference, JobProgressReport CheckoutModifiers = CheckoutModifiers.Force, }; - if (progressReporter != null) - { - var stage = $"Checkout {committish}"; - progressReporter = progressReporter.CreateSection(stage, 1.0); - progressReporter.ReportProgress(0); - checkoutOptions.OnCheckoutProgress = CheckoutProgressHandler(progressReporter); - } + var stage = $"Checkout {committish}"; + using var newProgressReporter = progressReporter.CreateSection(stage, 1.0); + newProgressReporter.ReportProgress(0); + checkoutOptions.OnCheckoutProgress = CheckoutProgressHandler(newProgressReporter); cancellationToken.ThrowIfCancellationRequested(); @@ -976,9 +999,38 @@ Task PushHeadToTemporaryBranch(string username, string password, JobProgressRepo try { var forcePushString = String.Format(CultureInfo.InvariantCulture, "+{0}:{0}", branch.CanonicalName); - libGitRepo.Network.Push(remote, forcePushString, GeneratePushOptions(progressReporter.CreateSection(null, 0.9), username, password, cancellationToken)); + + using (var mainPushReporter = progressReporter.CreateSection(null, 0.9)) + { + var (pushOptions, progressReporters) = GeneratePushOptions( + mainPushReporter, + username, + password, + cancellationToken); + + try + { + libGitRepo.Network.Push(remote, forcePushString, pushOptions); + } + finally + { + foreach (var progressReporter in progressReporters) + progressReporter.Dispose(); + } + } + var removalString = String.Format(CultureInfo.InvariantCulture, ":{0}", branch.CanonicalName); - libGitRepo.Network.Push(remote, removalString, GeneratePushOptions(progressReporter.CreateSection(null, 0.1), username, password, cancellationToken)); + using var forcePushReporter = progressReporter.CreateSection(null, 0.1); + var (forcePushOptions, forcePushReporters) = GeneratePushOptions(forcePushReporter, username, password, cancellationToken); + try + { + libGitRepo.Network.Push(remote, removalString, forcePushOptions); + } + finally + { + foreach (var subForcePushReporter in forcePushReporters) + forcePushReporter.Dispose(); + } } catch (UserCancelledException) { @@ -1005,32 +1057,39 @@ Task PushHeadToTemporaryBranch(string username, string password, JobProgressRepo /// The username for the . /// The password for the . /// The for the operation. - /// A new set of . - PushOptions GeneratePushOptions(JobProgressReporter progressReporter, string username, string password, CancellationToken cancellationToken) + /// A new set of and the associated s based off . + (PushOptions PushOptions, IEnumerable SubProgressReporters) GeneratePushOptions(JobProgressReporter progressReporter, string username, string password, CancellationToken cancellationToken) { - var subProgressReporter = progressReporter.CreateSection(null, 0.5); + var packFileCountingReporter = progressReporter.CreateSection(null, 0.25); + var packFileDeltafyingReporter = progressReporter.CreateSection(null, 0.25); + var transferProgressReporter = progressReporter.CreateSection(null, 0.5); - return new PushOptions - { - OnPackBuilderProgress = (stage, current, total) => - { - var baseProgress = stage == PackBuilderStage.Counting ? 0 : 0.5; - var addon = total > 0 && current <= total ? (0.5 * ((double)current / total)) : 0; - progressReporter.ReportProgress(baseProgress + addon); - return !cancellationToken.IsCancellationRequested; - }, - OnNegotiationCompletedBeforePush = (a) => + return ( + PushOptions: new PushOptions { - subProgressReporter = progressReporter.CreateSection(null, 0.5); - return !cancellationToken.IsCancellationRequested; + OnPackBuilderProgress = (stage, current, total) => + { + if (total < current) + total = current; + + var percentage = ((double)current) / total; + (stage == PackBuilderStage.Counting ? packFileCountingReporter : packFileDeltafyingReporter).ReportProgress(percentage); + return !cancellationToken.IsCancellationRequested; + }, + OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, + OnPushTransferProgress = (a, sentBytes, totalBytes) => + { + packFileCountingReporter.ReportProgress((double)sentBytes / totalBytes); + return !cancellationToken.IsCancellationRequested; + }, + CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), }, - OnPushTransferProgress = (a, sentBytes, totalBytes) => + SubProgressReporters: new List { - progressReporter.ReportProgress((double)sentBytes / totalBytes); - return !cancellationToken.IsCancellationRequested; - }, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), - }; + packFileCountingReporter, + packFileDeltafyingReporter, + transferProgressReporter, + }); } /// @@ -1054,7 +1113,7 @@ string GetRepositoryPath() /// The for the operation. /// A representing the running operation. ValueTask UpdateSubmodules( - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, string? username, string? password, bool deploymentPipeline, @@ -1062,7 +1121,7 @@ ValueTask UpdateSubmodules( { logger.LogTrace("Updating submodules {withOrWithout} credentials...", username == null ? "without" : "with"); - async ValueTask RecursiveUpdateSubmodules(LibGit2Sharp.IRepository parentRepository, JobProgressReporter? currentProgressReporter, string parentGitDirectory) + async ValueTask RecursiveUpdateSubmodules(LibGit2Sharp.IRepository parentRepository, JobProgressReporter currentProgressReporter, string parentGitDirectory) { var submoduleCount = libGitRepo.Submodules.Count(); if (submoduleCount == 0) @@ -1081,15 +1140,16 @@ async ValueTask RecursiveUpdateSubmodules(LibGit2Sharp.IRepository parentReposit OnCheckoutNotify = (_, _) => !cancellationToken.IsCancellationRequested, }; + using var fetchReporter = currentProgressReporter.CreateSection($"Fetch submodule {submodule.Name}", factor); + submoduleUpdateOptions.FetchOptions.Hydrate( logger, - currentProgressReporter?.CreateSection($"Fetch submodule {submodule.Name}", factor), + fetchReporter, credentialsProvider.GenerateCredentialsHandler(username, password), cancellationToken); - if (currentProgressReporter != null) - submoduleUpdateOptions.OnCheckoutProgress = CheckoutProgressHandler( - currentProgressReporter.CreateSection($"Checkout submodule {submodule.Name}", factor)); + using var checkoutReporter = currentProgressReporter.CreateSection($"Checkout submodule {submodule.Name}", factor); + submoduleUpdateOptions.OnCheckoutProgress = CheckoutProgressHandler(checkoutReporter); logger.LogDebug("Updating submodule {submoduleName}...", submodule.Name); Task RawSubModuleUpdate() => Task.Factory.StartNew( @@ -1106,7 +1166,7 @@ Task RawSubModuleUpdate() => Task.Factory.StartNew( { // workaround for https://github.com/libgit2/libgit2/issues/3820 // kill off the modules/ folder in .git and try again - currentProgressReporter?.ReportProgress(null); + currentProgressReporter.ReportProgress(0); credentialsProvider.CheckBadCredentialsException(ex); logger.LogWarning(ex, "Initial update of submodule {submoduleName} failed. Deleting submodule directories and re-attempting...", submodule.Name); @@ -1145,9 +1205,11 @@ await eventConsumer.HandleEvent( using var submoduleRepo = await submoduleFactory.CreateFromPath( submodulePath, cancellationToken); + + using var submoduleReporter = currentProgressReporter.CreateSection($"Entering submodule \"{submodule.Name}\"...", factor); await RecursiveUpdateSubmodules( submoduleRepo, - currentProgressReporter?.CreateSection($"Entering submodule \"{submodule.Name}\"...", factor), + submoduleReporter, submodulePath); } } diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 94ae1769510..74a6bc67514 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -123,7 +123,7 @@ public void Dispose() string? initialBranch, string? username, string? password, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, bool recurseSubmodules, CancellationToken cancellationToken) { @@ -146,8 +146,8 @@ public void Dispose() if (!await ioManager.DirectoryExists(repositoryPath, cancellationToken)) try { - var cloneProgressReporter = progressReporter?.CreateSection(null, 0.75f); - var checkoutProgressReporter = progressReporter?.CreateSection(null, 0.25f); + using var cloneProgressReporter = progressReporter.CreateSection(null, 0.75f); + using var checkoutProgressReporter = progressReporter.CreateSection(null, 0.25f); var cloneOptions = new CloneOptions { RecurseSubmodules = recurseSubmodules, diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs index 9861a854183..17e820433a7 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs @@ -173,10 +173,7 @@ public async ValueTask RepositoryUpdateJob( var numSteps = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0) + (!modelHasShaOrReference ? 2 : (hardResettingToOriginReference ? 3 : 1)); var progressFactor = 1.0 / numSteps; - JobProgressReporter NextProgressReporter(string? stage) - { - return progressReporter.CreateSection(stage, progressFactor); - } + JobProgressReporter NextProgressReporter(string? stage) => progressReporter.CreateSection(stage, progressFactor); progressReporter.ReportProgress(0); @@ -246,29 +243,35 @@ ValueTask CallLoadRevInfo(Models.TestMerge? testMergeToAdd = null, string? lastO { if (!repo.Tracking) throw new JobException(ErrorCode.RepoReferenceRequired); - await repo.FetchOrigin( - NextProgressReporter("Fetch Origin"), - currentModel.AccessUser, - currentModel.AccessToken, - false, - cancellationToken); + using (var fetchReporter = NextProgressReporter("Fetch Origin")) + await repo.FetchOrigin( + fetchReporter, + currentModel.AccessUser, + currentModel.AccessToken, + false, + cancellationToken); if (!modelHasShaOrReference) { - var fastForward = await repo.MergeOrigin( - NextProgressReporter("Merge Origin"), - committerName, - currentModel.CommitterEmail!, - false, - cancellationToken); + bool? fastForward; + using (var mergeReporter = NextProgressReporter("Merge Origin")) + fastForward = await repo.MergeOrigin( + mergeReporter, + committerName, + currentModel.CommitterEmail!, + false, + cancellationToken); + if (!fastForward.HasValue) throw new JobException(ErrorCode.RepoMergeConflict); + lastRevisionInfo!.OriginCommitSha = await repo.GetOriginSha(cancellationToken); await UpdateRevInfo(); if (fastForward.Value) { + using var syncReporter = NextProgressReporter("Sychronize"); await repo.Synchronize( - NextProgressReporter("Sychronize"), + syncReporter, currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName!, @@ -279,7 +282,7 @@ await repo.Synchronize( postUpdateSha = repo.Head; } else - NextProgressReporter(null).ReportProgress(1.0); + NextProgressReporter(null).Dispose(); } } @@ -303,39 +306,44 @@ await repo.Synchronize( if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null)) throw new JobException(ErrorCode.RepoSwappedShaOrReference); - await repo.CheckoutObject( - committish, - currentModel.AccessUser, - currentModel.AccessToken, - updateSubmodules, - false, - NextProgressReporter("Checkout"), - cancellationToken); + using (var checkoutReporter = NextProgressReporter("Checkout")) + await repo.CheckoutObject( + committish, + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + false, + checkoutReporter, + cancellationToken); await CallLoadRevInfo(); // we've either seen origin before or what we're checking out is on origin } else - NextProgressReporter(null).ReportProgress(1.0); + NextProgressReporter(null).Dispose(); if (hardResettingToOriginReference) { if (!repo.Tracking) throw new JobException(ErrorCode.RepoReferenceNotTracking); - await repo.ResetToOrigin( - NextProgressReporter("Reset to Origin"), - currentModel.AccessUser, - currentModel.AccessToken, - updateSubmodules, - false, - cancellationToken); - await repo.Synchronize( - NextProgressReporter("Synchronize"), - currentModel.AccessUser, - currentModel.AccessToken, - currentModel.CommitterName!, - currentModel.CommitterEmail!, - true, - false, - cancellationToken); + using (var resetReporter = NextProgressReporter("Reset to Origin")) + await repo.ResetToOrigin( + resetReporter, + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + false, + cancellationToken); + + using (var syncReporter = NextProgressReporter("Synchronize")) + await repo.Synchronize( + syncReporter, + currentModel.AccessUser, + currentModel.AccessToken, + currentModel.CommitterName!, + currentModel.CommitterEmail!, + true, + false, + cancellationToken); + await CallLoadRevInfo(); // repo head is on origin so force this @@ -486,7 +494,8 @@ await databaseContextFactory.UseContext( // goteem var commitSha = revInfoWereLookingFor.CommitSha!; logger.LogDebug("Reusing existing SHA {sha}...", commitSha); - await repo.ResetToSha(commitSha, NextProgressReporter($"Reset to {commitSha[..7]}"), cancellationToken); + using var resetReporter = NextProgressReporter($"Reset to {commitSha[..7]}"); + await repo.ResetToSha(commitSha, resetReporter, cancellationToken); lastRevisionInfo = revInfoWereLookingFor; } @@ -499,15 +508,17 @@ await databaseContextFactory.UseContext( var fullTestMergeTask = repo.GetTestMerge(newTestMerge, currentModel, cancellationToken); - var mergeResult = await repo.AddTestMerge( - newTestMerge, - committerName, - currentModel.CommitterEmail!, - currentModel.AccessUser, - currentModel.AccessToken, - updateSubmodules, - NextProgressReporter($"Test merge #{newTestMerge.Number}"), - cancellationToken); + TestMergeResult mergeResult; + using (var testMergeReporter = NextProgressReporter($"Test merge #{newTestMerge.Number}")) + mergeResult = await repo.AddTestMerge( + newTestMerge, + committerName, + currentModel.CommitterEmail!, + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + testMergeReporter, + cancellationToken); if (mergeResult.Status == MergeStatus.Conflicts) throw new JobException( @@ -546,15 +557,17 @@ await databaseContextFactory.UseContext( var currentHead = repo.Head; if (currentModel.PushTestMergeCommits!.Value && (startSha != currentHead || (postUpdateSha != null && postUpdateSha != currentHead))) { - await repo.Synchronize( - NextProgressReporter("Synchronize"), - currentModel.AccessUser, - currentModel.AccessToken, - currentModel.CommitterName!, - currentModel.CommitterEmail!, - false, - false, - cancellationToken); + using (var syncReporter = NextProgressReporter("Synchronize")) + await repo.Synchronize( + syncReporter, + currentModel.AccessUser, + currentModel.AccessToken, + currentModel.CommitterName!, + currentModel.CommitterEmail!, + false, + false, + cancellationToken); + await UpdateRevInfo(); } } @@ -568,17 +581,19 @@ await repo.Synchronize( var secondStep = startReference != null && repo.Head != startSha; // DCTx2: Cancellation token is for job, operations should always run - await repo.CheckoutObject( - startReference ?? startSha, - currentModel.AccessUser, - currentModel.AccessToken, - true, - false, - progressReporter.CreateSection($"Checkout {startReference ?? startSha[..7]}", secondStep ? 0.5 : 1.0), - default); + using (var checkoutReporter = progressReporter.CreateSection($"Checkout {startReference ?? startSha[..7]}", secondStep ? 0.5 : 1.0)) + await repo.CheckoutObject( + startReference ?? startSha, + currentModel.AccessUser, + currentModel.AccessToken, + true, + false, + checkoutReporter, + default); if (secondStep) - await repo.ResetToSha(startSha, progressReporter.CreateSection($"Hard reset to SHA {startSha[..7]}", 0.5), default); + using (var resetReporter = progressReporter.CreateSection($"Hard reset to SHA {startSha[..7]}", 0.5)) + await repo.ResetToSha(startSha, resetReporter, default); throw; } @@ -608,32 +623,35 @@ public async ValueTask RepositoryRecloneJob( string? oldReference; string oldSha; ValueTask deleteTask; - using (var oldRepo = await instance.RepositoryManager.LoadRepository(cancellationToken)) + using (var deleteReporter = progressReporter.CreateSection("Deleting Old Repository", 0.1)) { - if (oldRepo == null) - throw new JobException(ErrorCode.RepoMissing); + using (var oldRepo = await instance.RepositoryManager.LoadRepository(cancellationToken)) + { + if (oldRepo == null) + throw new JobException(ErrorCode.RepoMissing); + + origin = oldRepo.Origin; + oldSha = oldRepo.Head; + oldReference = oldRepo.Reference; + if (oldReference == Repository.NoReference) + oldReference = null; - origin = oldRepo.Origin; - oldSha = oldRepo.Head; - oldReference = oldRepo.Reference; - if (oldReference == Repository.NoReference) - oldReference = null; + deleteTask = instance.RepositoryManager.DeleteRepository(cancellationToken); + } - progressReporter.StageName = "Deleting Old Repository"; - deleteTask = instance.RepositoryManager.DeleteRepository(cancellationToken); + await deleteTask; } - await deleteTask; - progressReporter.ReportProgress(0.1); IRepository newRepo; try { + using var cloneReporter = progressReporter.CreateSection("Cloning New Repository", 0.8); newRepo = await instance.RepositoryManager.CloneRepository( origin, oldReference, currentModel.AccessUser, currentModel.AccessToken, - progressReporter.CreateSection("Cloning New Repository", 0.8), + cloneReporter, true, // TODO: Make configurable maybe... cancellationToken) ?? throw new JobException("A race condition occurred while recloning the repository. Somehow, it was fully cloned instantly after being deleted!"); // I'll take lines of code that should never be hit for $10k @@ -655,14 +673,17 @@ await databaseContextFactory.UseContextTaskReturn(context => } using (newRepo) + using (var checkoutReporter = progressReporter.CreateSection("Checking out previous Detached Commit", 0.1)) + { await newRepo.CheckoutObject( oldSha, currentModel.AccessUser, currentModel.AccessToken, false, oldReference != null, - progressReporter.CreateSection("Checking out previous Detached Commit", 0.1), + checkoutReporter, cancellationToken); + } } } } diff --git a/src/Tgstation.Server.Host/Controllers/EngineController.cs b/src/Tgstation.Server.Host/Controllers/EngineController.cs index c1a20fe99ca..83a113e3b46 100644 --- a/src/Tgstation.Server.Host/Controllers/EngineController.cs +++ b/src/Tgstation.Server.Host/Controllers/EngineController.cs @@ -184,7 +184,8 @@ public async ValueTask Update([FromBody] EngineVersionRequest mod try { - await byondManager.ChangeVersion(null, model.EngineVersion, null, false, cancellationToken); + using var progressReporter = new JobProgressReporter(); + await byondManager.ChangeVersion(progressReporter, model.EngineVersion, null, false, cancellationToken); } catch (InvalidOperationException ex) { diff --git a/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs b/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs index 6d71a68edf8..b5e996c311c 100644 --- a/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs @@ -20,14 +20,14 @@ static class FetchOptionsExtensions /// /// The to hydrate. /// The for the operation. - /// The optional . + /// The . /// The optional . /// The for the operation. /// The hydrated . public static FetchOptions Hydrate( this FetchOptions fetchOptions, ILogger logger, - JobProgressReporter? progressReporter, + JobProgressReporter progressReporter, CredentialsHandler credentialsHandler, CancellationToken cancellationToken) { @@ -60,10 +60,10 @@ public static FetchOptions Hydrate( /// Generate a from a given and . /// /// The for the operation. - /// The optional of the operation. + /// The of the operation. /// The for the operation. /// A new based on . - static TransferProgressHandler TransferProgressHandler(ILogger logger, JobProgressReporter? progressReporter, CancellationToken cancellationToken) => transferProgress => + static TransferProgressHandler TransferProgressHandler(ILogger logger, JobProgressReporter progressReporter, CancellationToken cancellationToken) => transferProgress => { double? percentage; var totalObjectsToProcess = transferProgress.TotalObjects * 2; diff --git a/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs b/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs index 753f8cf519e..776a2c19e79 100644 --- a/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs +++ b/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs @@ -1,6 +1,7 @@ using System; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Tgstation.Server.Host.Models; @@ -9,7 +10,7 @@ namespace Tgstation.Server.Host.Jobs /// /// Progress reporter for a . /// - public sealed class JobProgressReporter + public sealed class JobProgressReporter : IDisposable { /// /// The name of the current stage. @@ -52,6 +53,24 @@ public string? StageName /// double sectionProgression; + /// + /// The total progress reserved for use in this section. + /// + double? sectionReservations; + + /// + /// Initializes a new instance of the class. + /// + /// This variant has no function. + public JobProgressReporter() + : this( + NullLogger.Instance, + null, + (_, _) => { }, + false) + { + } + /// /// Initializes a new instance of the class. /// @@ -59,27 +78,84 @@ public string? StageName /// The value of . /// The value of . public JobProgressReporter(ILogger logger, string? stageName, Action callback) + : this( + logger, + stageName, + callback, + true) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + /// If an initial call to will be made with only the . + private JobProgressReporter(ILogger logger, string? stageName, Action callback, bool setStageName) { this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.callback = callback ?? throw new ArgumentNullException(nameof(callback)); - StageName = stageName; + if (setStageName) + { + StageName = stageName; + } + else + { + this.stageName = stageName; + } logger.LogDebug("Job progress reporter created. Stage: {stageName}", stageName ?? "(null)"); } + /// + public void Dispose() + { + if (sectionReservations.HasValue) + if (sectionReservations.Value != 1.0) + { + // not an error, processes can throw + sectionReservations = null; + } + else if (sectionProgression < 1.0) + { + logger.LogError( + new InvalidOperationException($"Parent progress reporter has child sections that didn't complete! Current: {sectionProgression}"), + "TGS BUG: Progress reporter children didn't complete!"); + sectionReservations = null; + } + + if (!sectionReservations.HasValue) + ReportProgress(1); + } + /// /// Report progress. /// /// A percentage value from 0.0f-1.0f. public void ReportProgress(double? progress) { + if (sectionReservations.HasValue) + if (progress == 0) + { + // might be a stage reset + sectionReservations = null; + } + else + { + logger.LogError( + new InvalidOperationException("Progress reporter is reporting progress with existing nested sections!"), + "TGS BUG: A progress reporter is using mixed local and nested progress, this is not supported"); + } + var clampedProgress = progress; if (progress.HasValue) if (progress > 1 || progress < 0) { logger.LogError( new ArgumentOutOfRangeException(nameof(progress), progress, "Progress must be a value from 0-1!"), - "Invalid progress value for stage {stageName}", + "TGS BUG: Invalid progress value for stage {stageName}", StageName ?? "(null)"); clampedProgress = null; } @@ -103,16 +179,27 @@ public JobProgressReporter CreateSection(string? newStageName, double percentage { logger.LogError( new ArgumentOutOfRangeException(nameof(percentage), percentage, "Percentage must be a value from 0-1!"), - "Invalid percentage value for stage {newStageName}! Clamping...", + "TGS BUG: Invalid percentage value for stage {newStageName}! Clamping...", newStageName ?? "(null)"); percentage = Math.Min(Math.Max(percentage, 0.0), 1.0); } - var childBaseProgress = sectionProgression; - if (percentage + childBaseProgress > 1.0) + if (!sectionReservations.HasValue) { - var remainingPercentage = 1.0 - childBaseProgress; + if (sectionProgression != 0) + { + logger.LogError( + new InvalidOperationException("Progress reporter is creating a section with local progress!"), + "TGS BUG: A progress reporter is using mixed local and nested progress, this is not supported"); + } + + sectionReservations = 0; + } + + if (percentage + sectionReservations.Value > 1.0001) // floating point >.< + { + var remainingPercentage = 1.0 - sectionReservations.Value; logger.LogError( "Stage {newStageName} is overbudgeted ({budget}/{remainingPercentage})! Clamping...", newStageName, @@ -121,6 +208,9 @@ public JobProgressReporter CreateSection(string? newStageName, double percentage percentage = remainingPercentage; } + Math.Min(sectionReservations.Value + percentage, 1); + + var childLocalProgress = 0.0; var newReporter = new JobProgressReporter( logger, newStageName, @@ -133,11 +223,17 @@ public JobProgressReporter CreateSection(string? newStageName, double percentage return; } - var childLocalProgress = progress.Value * percentage; + var progressWithoutChild = sectionProgression - childLocalProgress; + childLocalProgress = progress.Value * percentage; + + // floating point >.< + sectionProgression = Math.Min(progressWithoutChild + childLocalProgress, 1); + if (sectionProgression > 9.9999) + sectionProgression = 1; - sectionProgression = childLocalProgress + childBaseProgress; callback(currentStage, sectionProgression); - }); + }, + false); newReporter.ReportProgress(0); return newReporter; diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index e9967f5189c..d78cc7a2872 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -460,14 +460,16 @@ void UpdateProgress(string? stage, double? progress) QueueHubUpdate(job.ToApi(), false); logger.LogTrace("Starting job..."); + using var progressReporter = new JobProgressReporter( + loggerFactory.CreateLogger(), + null, + UpdateProgress); + using var innerReporter = progressReporter.CreateSection(null, 1.0); await operation( instanceCoreProvider.GetInstance(job.Instance!), databaseContextFactory, job, - new JobProgressReporter( - loggerFactory.CreateLogger(), - null, - UpdateProgress), + innerReporter, cancellationToken); logger.LogDebug("Job {jobId} completed!", job.Id); diff --git a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs index 2a62fea29ad..8950acc6ab6 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs @@ -13,6 +13,7 @@ using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; @@ -52,7 +53,7 @@ static async Task RepoDownloadTest(bool needsClone) null, null, null, - null, + It.IsNotNull(), true, It.IsAny())) .Callback(() => ++cloneAttempts) @@ -85,7 +86,7 @@ static async Task RepoDownloadTest(bool needsClone) Engine = EngineType.OpenDream, SourceSHA = new string('a', Limits.MaximumCommitShaLength), }, - null, + new JobProgressReporter(), CancellationToken.None); diff --git a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs index c4a17f51401..2cdd59d19b8 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryManager.cs @@ -15,6 +15,7 @@ using Tgstation.Server.Host.Components.Events; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; namespace Tgstation.Server.Host.Components.Repository.Tests { @@ -86,7 +87,7 @@ public async Task TestBasicClone() null, null, null, - null, + new JobProgressReporter(), false, CancellationToken.None); diff --git a/tests/Tgstation.Server.Host.Tests/Jobs/TestJobProgressReporter.cs b/tests/Tgstation.Server.Host.Tests/Jobs/TestJobProgressReporter.cs new file mode 100644 index 00000000000..f6d6cd9348f --- /dev/null +++ b/tests/Tgstation.Server.Host.Tests/Jobs/TestJobProgressReporter.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using Moq; + +namespace Tgstation.Server.Host.Jobs.Tests +{ + [TestClass] + public sealed class TestJobProgressReporter + { + string expectedStageName = null; + double? expectedProgress = null; + void Validate(string stageName, double? progress) + { + Assert.AreEqual(expectedStageName, stageName); + Assert.AreEqual(expectedProgress, progress); + } + + JobProgressReporter Setup() + { + expectedStageName = null; + expectedProgress = 0; + return new JobProgressReporter( + Mock.Of>(), + null, + Validate); + } + + [TestMethod] + public void TestBasicUsage() + { + var progressReporter = Setup(); + + expectedProgress = 0.4; + progressReporter.ReportProgress(0.4); + expectedProgress = 1.0; + progressReporter.ReportProgress(1.0); + } + + [TestMethod] + public void TestNestedUsage() + { + var progressReporter = Setup(); + + expectedStageName = "Test1"; + var subReporter1 = progressReporter.CreateSection("Test1", 0.5); + expectedProgress = 0.1; + subReporter1.ReportProgress(0.2); + expectedProgress = 0.4; + subReporter1.ReportProgress(0.8); + + expectedStageName = "Test2"; + var subReporter2 = progressReporter.CreateSection("Test2", 0.5); + + expectedStageName = "Test1"; + expectedProgress = 0.5; + subReporter1.ReportProgress(1); + + expectedStageName = "Test2"; + expectedProgress = 0.6; + subReporter2.ReportProgress(0.2); + expectedProgress = 1.0; + subReporter2.ReportProgress(1); + } + } +} diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 9c2b44c9624..67ed9e2178f 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -22,6 +22,7 @@ using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; @@ -142,7 +143,7 @@ public static async ValueTask DownloadEngineVersion( using var windowsByondInstaller = byondInstaller as WindowsByondInstaller; // get the bytes for stable - return await byondInstaller.DownloadVersion(compatVersion, null, cancellationToken); + return await byondInstaller.DownloadVersion(compatVersion, new JobProgressReporter(), cancellationToken); } public async Task RunCompatTests( From 1c3a99ab6a02fa678827590611235cfee09d8b02 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 18 Aug 2024 00:00:14 -0400 Subject: [PATCH 105/111] Fix issues with retrieving process memory --- .../Components/Session/SessionController.cs | 2 +- src/Tgstation.Server.Host/System/IProcessBase.cs | 2 +- src/Tgstation.Server.Host/System/Process.cs | 16 +++++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 6880ed7f6d2..754b672b7d6 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -122,7 +122,7 @@ async Task Wrap() public FifoSemaphore TopicSendSemaphore { get; } /// - public long MemoryUsage => process.MemoryUsage; + public long? MemoryUsage => process.MemoryUsage; /// /// The for the . diff --git a/src/Tgstation.Server.Host/System/IProcessBase.cs b/src/Tgstation.Server.Host/System/IProcessBase.cs index f34d5e6597c..6ae88dce502 100644 --- a/src/Tgstation.Server.Host/System/IProcessBase.cs +++ b/src/Tgstation.Server.Host/System/IProcessBase.cs @@ -16,7 +16,7 @@ public interface IProcessBase /// /// Gets the process' memory usage in bytes. /// - long MemoryUsage { get; } + long? MemoryUsage { get; } /// /// Set's the owned to a non-normal value. diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index 64c92bf89b9..13a1a278f50 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -23,7 +23,21 @@ sealed class Process : IProcess public Task Lifetime { get; } /// - public long MemoryUsage => handle.VirtualMemorySize64; + public long? MemoryUsage + { + get + { + try + { + return handle.VirtualMemorySize64; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to get PID {pid}'s memory usage!", Id); + return null; + } + } + } /// /// The for the . From bab11e603aa2d03f0432b437fd10778cce5f624f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 18 Aug 2024 00:26:59 -0400 Subject: [PATCH 106/111] Fix Release build --- src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs b/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs index 776a2c19e79..12ee246858f 100644 --- a/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs +++ b/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs @@ -197,7 +197,8 @@ public JobProgressReporter CreateSection(string? newStageName, double percentage sectionReservations = 0; } - if (percentage + sectionReservations.Value > 1.0001) // floating point >.< + // floating point >.< + if (percentage + sectionReservations.Value > 1.0001) { var remainingPercentage = 1.0 - sectionReservations.Value; logger.LogError( From b71e3f407e7c77d38b30a25b7b521a4de66a8ee3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 18 Aug 2024 08:50:36 -0400 Subject: [PATCH 107/111] Make re-run flaky tests work with CI Security workflow --- .github/workflows/rerun-flaky-tests.yml | 2 +- .github/workflows/scripts/rerunFlakyTests.js | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rerun-flaky-tests.yml b/.github/workflows/rerun-flaky-tests.yml index 9016637a498..806d89e3597 100644 --- a/.github/workflows/rerun-flaky-tests.yml +++ b/.github/workflows/rerun-flaky-tests.yml @@ -1,7 +1,7 @@ name: Rerun Flaky Live Tests on: workflow_run: - workflows: [CI Pipeline] + workflows: ['CI Pipeline', 'CI Security'] types: - completed jobs: diff --git a/.github/workflows/scripts/rerunFlakyTests.js b/.github/workflows/scripts/rerunFlakyTests.js index 589d6d85c99..d1919838782 100644 --- a/.github/workflows/scripts/rerunFlakyTests.js +++ b/.github/workflows/scripts/rerunFlakyTests.js @@ -38,7 +38,9 @@ export async function rerunFlakyTests({ github, context }) { const filteredFailingJobs = failingJobs.filter((job) => { console.log(`Failing job: ${job.name}`) - return CONSIDERED_JOBS.some((title) => job.name.startsWith(title)); + return CONSIDERED_JOBS + .flatMap(jobName => [jobName, 'CI Pipeline / ' + jobName]) + .some((title) => job.name.startsWith(title)); }); if (filteredFailingJobs.length !== failingJobs.length) { console.log("One or more failing jobs are NOT designated flaky. Not rerunning."); From 4903dacf3f5ed2e298a529c831cb7816cc073a70 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 18 Aug 2024 08:59:49 -0400 Subject: [PATCH 108/111] Add size labeller workflow --- .github/workflows/size-label.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/size-label.yml diff --git a/.github/workflows/size-label.yml b/.github/workflows/size-label.yml new file mode 100644 index 00000000000..8359d7c43ad --- /dev/null +++ b/.github/workflows/size-label.yml @@ -0,0 +1,19 @@ +name: Size Labelling +on: + pull_request_target: + branches: + - dev + - master + +jobs: + size-label: + name: Add Size Label + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: size-label + uses: "pascalgn/size-label-action@bbbaa0d5ccce8e2e76254560df5c64b82dac2e12" # v0.5.2, consider upgrading after https://github.com/pascalgn/size-label-action/pull/54 is merged + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 426d53a06e8d635a182fca5fa5a81aa758a86aa1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 18 Aug 2024 09:04:26 -0400 Subject: [PATCH 109/111] Trigger PR labeller on all PRs --- .github/workflows/size-label.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/size-label.yml b/.github/workflows/size-label.yml index 8359d7c43ad..096ce4c6432 100644 --- a/.github/workflows/size-label.yml +++ b/.github/workflows/size-label.yml @@ -1,9 +1,6 @@ name: Size Labelling on: pull_request_target: - branches: - - dev - - master jobs: size-label: From d82b5b5bd1b87538c3c166004c63a988eebbed7e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 18 Aug 2024 12:22:01 -0400 Subject: [PATCH 110/111] Add missing ptrace capability to systemd service --- build/tgstation-server.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/tgstation-server.service b/build/tgstation-server.service index 6f3c087584b..98c03fee57c 100644 --- a/build/tgstation-server.service +++ b/build/tgstation-server.service @@ -17,7 +17,7 @@ Restart=always KillMode=process ReloadSignal=SIGUSR2 RestartKillSignal=SIGUSR2 -AmbientCapabilities=CAP_SYS_NICE +AmbientCapabilities=CAP_SYS_NICE CAP_SYS_PTRACE WatchdogSec=60 WatchdogSignal=SIGTERM From ec71b37a1da0d71e90448495259479083d092070 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 18 Aug 2024 12:49:04 -0400 Subject: [PATCH 111/111] CodeCov uploading sanity pass - Upgrade action to v4. - Add ReleaseNotes check to fail job if it doesn't detect the `codecov/project` check on the run in 15 minutes. - Mark Upload Codecov Job as flaky. --- .github/workflows/ci-pipeline.yml | 17 ++++++++++--- .github/workflows/scripts/rerunFlakyTests.js | 3 ++- .../Tgstation.Server.ReleaseNotes/Program.cs | 25 ++++++++++++++++++- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 08a7a1cfc53..e70b298c560 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -41,7 +41,6 @@ env: TGS_TEST_GITHUB_TOKEN: ${{ secrets.LIVE_TESTS_TOKEN }} TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} PACKAGING_PRIVATE_KEY_PASSPHRASE: ${{ secrets.PACKAGING_PRIVATE_KEY_PASSPHRASE }} - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} concurrency: group: "ci-${{ (github.event_name != 'push' && github.event_name != 'schedule' && github.event.inputs.pull_request_number) || github.run_id }}-${{ github.event_name }}" @@ -967,7 +966,7 @@ jobs: upload-code-coverage: name: Upload Code Coverage - needs: [ linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-tests ] + needs: [ linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-tests, build-releasenotes ] runs-on: ubuntu-latest steps: - name: Checkout (Branch) @@ -1214,11 +1213,23 @@ jobs: name: windows-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlite + - name: Retrieve ReleaseNotes Binaries + uses: actions/download-artifact@v4 + with: + name: release_notes_bins + path: release_notes_bins + - name: Upload Coverage to CodeCov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: directory: ./code_coverage fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + handle_no_reports_found: true + + - name: Wait for CodeCov Status + run: dotnet release_notes_bins/Tgstation.Server.ReleaseNotes --wait-codecov ${{ github.run_id }} build-deb: name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595 diff --git a/.github/workflows/scripts/rerunFlakyTests.js b/.github/workflows/scripts/rerunFlakyTests.js index d1919838782..7dda0f389ad 100644 --- a/.github/workflows/scripts/rerunFlakyTests.js +++ b/.github/workflows/scripts/rerunFlakyTests.js @@ -3,7 +3,8 @@ const CONSIDERED_JOBS = [ "Windows Live Tests", "Linux Live Tests", - "Build .deb Package" + "Build .deb Package", + "Upload Code Coverage" ]; async function getFailedJobsForRun(github, context, workflowRunId, runAttempt) { diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index 46c887fd5f6..c59f86fd19b 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -60,6 +60,7 @@ static async Task Main(string[] args) var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase); var genToken = versionString.Equals("--token-output-file", StringComparison.OrdinalIgnoreCase); + var waitCodecov = versionString.Equals("--wait-codecov", StringComparison.OrdinalIgnoreCase); if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease @@ -67,7 +68,8 @@ static async Task Main(string[] args) && !shaCheck && !fullNotes && !nuget - && !genToken) + && !genToken + && !waitCodecov) { Console.WriteLine("Invalid version: " + versionString); return 2; @@ -155,6 +157,11 @@ static async Task Main(string[] args) client.Credentials = new Credentials(githubToken); } + if (waitCodecov) + { + return await CodecovCheck(client, Int64.Parse(args[1])); + } + if (linkWinget) { if (args.Length < 2 || !Uri.TryCreate(args[1], new UriCreationOptions(), out var actionsUrl)) @@ -1671,5 +1678,21 @@ static void DebugAssert(bool condition, string message = null) else Debug.Assert(condition); } + + static async ValueTask CodecovCheck(IGitHubClient client, long runId) + { + var currentRun = await client.Actions.Workflows.Runs.Get(RepoOwner, RepoName, runId); + + bool foundRun = false; + for(int i = 0; i < 15 && !foundRun; ++i) + { + var allRuns = await client.Check.Run.GetAllForReference(RepoOwner, RepoName, currentRun.HeadSha); + foundRun = allRuns.CheckRuns.Any(x => x.CheckSuite.Id == currentRun.Id && x.Name == "codecov/project"); + if (!foundRun && i != 14) + await Task.Delay(TimeSpan.FromMinutes(1)); + } + + return foundRun ? 0 : 24398; + } } }