Common Build Tooling (#55373)

Add Common Build Tooling

## Information for Developers

On Windows, the build scripts will automatically install Node. Other 
platforms should use their package manager or download it from 
https://nodejs.org/en/download/

Pick one:

- VSCode: `Ctrl+Shift+B`
- VSCode: `F5` (build & run with debugger) or `Ctrl+F5` (build & run 
  without debugger)
- Windows: double-click root `Build.bat` (pause to see output) or 
  `tools/build/build.bat` (no pause)
- Git Bash and non-Windows: `tools/build/build`

## Information for Server Admins

- TGS scripts will automatically install the version of Node specified 
  in `dependencies.sh`
- Either use this build script, or compile tgui by running any script 
  in `tgui/bin` folder.

## Details

Both dm and tgui are now built with a single script. It's pretty easy 
to launch: just press `Ctrl+Shift+B` in VSCode, and tada! 🎉 

**It's smart.** It will skip certain steps if source files were 
untouched for that step. So, if you're only touching dm code, it will 
only rebuild dm code, and will skip tgui.

**Syntax is fairly readable and maintainable.**

```js
const { Task, runTasks, exec } = require('./cbt');

const taskTgui = new Task('tgui')
  .depends('tgui/yarn.lock')
  .depends('tgui/packages/**/*.js')
  .provides('tgui/public/*.bundle.*')
  .provides('tgui/public/*.chunk.*')
  .build(async () => {
    await exec('tgui/bin/tgui');
  });

runTasks([taskTgui]);
```

**This is a long term solution to the js bundle hell.** Now that we 
have a single script to build everything, bundles have been excluded 
from the repo, and they will no longer cause conflicts in PRs. This 
results in quicker PR turnaround time and less time wasted on 
rebuilding tgui for PRs.

**CI pipelines have been updated.** They're not coded in the most 
optimal way, just making them green for now.

## Possible future work

- Support compiling with DM defines by passing them as an argument, 
  like `-D LOWMEMORYMODE`.
- Instead of explicitly listing the task sequence in `runTasks()`, 
  support specifying tasks in `.depends()`, which in turn will allow 
  building a graph of dependencies and running things in parallel.

Co-authored-by: Tad Hardesty <[email protected]>

- Add root Build.bat
- Add trio of tools/bootstrap/ scripts for Node
- Add tools/build/README.md
- Ensure build script and VSC configuration works properly on Linux
- Update TGS4 PreCompile scripts

Co-authored-by: Jordan Brown <[email protected]>

- Keep PreCompile scripts compatible with TGS3 as well
- Update LinuxOneShot PreCompile scripts
- Update TGS4 configuration

.github/workflows/ci_suite.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install Tools
         run: |
           pip3 install setuptools
-          bash tools/ci/install_build_tools.sh
+          bash tools/ci/install_node.sh
           bash tools/ci/install_spaceman_dmm.sh dreamchecker
           tools/bootstrap/python -c ''
       - name: Run Linters
@@ -30,7 +30,7 @@ jobs:
           bash tools/ci/check_changelogs.sh
           find . -name "*.php" -print0 | xargs -0 -n1 php -l
           find . -name "*.json" -not -path "*/node_modules/*" -print0 | xargs -0 python3 ./tools/json_verifier.py
-          bash tools/ci/build_tgui.sh
+          tgui/bin/tgui --lint
           bash tools/ci/check_grep.sh
           tools/bootstrap/python -m dmi.test
           tools/bootstrap/python -m mapmerge2.dmm_test
@@ -57,7 +57,9 @@ jobs:
           bash tools/ci/install_byond.sh
           source $HOME/BYOND/byond/bin/byondsetup
           python3 tools/ci/template_dm_generator.py
+          tgui/bin/tgui --build
           bash tools/ci/dm.sh -DCIBUILDING -DCITESTING -DALL_MAPS tgstation.dme
+
   run_all_tests:
     if: "!contains(github.event.head_commit.message, '[ci skip]')"
     name: Integration Tests
@@ -94,8 +96,10 @@ jobs:
         run: |
           bash tools/ci/install_byond.sh
           source $HOME/BYOND/byond/bin/byondsetup
+          tgui/bin/tgui --build
           bash tools/ci/dm.sh -DCIBUILDING tgstation.dme
           bash tools/ci/run_server.sh
+
   test_windows:
     if: "!contains(github.event.head_commit.message, '[ci skip]')"
     name: Windows Build

.gitignore

@@ -231,3 +231,6 @@ tools/LinuxOneShot/Database
 tools/LinuxOneShot/TGS_Config
 tools/LinuxOneShot/TGS_Instances
 tools/LinuxOneShot/TGS_Logs
+
+# Common build tooling
+!/tools/build

.tgs4.yml

@@ -4,3 +4,5 @@ static_files:
   - name: data
 linux_scripts:
   PreCompile.sh: tools/tgs4_scripts/PreCompile.sh
+windows_scripts:
+  PreCompile.bat: tools/tgs4_scripts/PreCompile.bat

.vscode/launch.json

@@ -0,0 +1,12 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "byond",
+      "request": "launch",
+      "name": "Launch DreamSeeker",
+      "preLaunchTask": "Build All",
+      "dmb": "${workspaceFolder}/${command:CurrentDMB}"
+    }
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,39 @@
+{
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"type": "process",
+			"command": "tools/build/build",
+			"windows": {
+				"command": ".\\tools\\build\\build.bat"
+			},
+			"problemMatcher": [
+				"$dreammaker",
+				"$eslint-stylish"
+			],
+			"group": "build",
+			"label": "Build All"
+		},
+		{
+			"type": "dreammaker",
+			"dme": "tgstation.dme",
+			"problemMatcher": [
+				"$dreammaker"
+			],
+			"group": "build",
+			"label": "dm: build - tgstation.dme"
+		},
+		{
+			"type": "shell",
+			"command": "tgui/bin/tgui",
+			"windows": {
+				"command": ".\\tgui\\bin\\tgui.bat"
+			},
+			"problemMatcher": [
+				"$eslint-stylish"
+			],
+			"group": "build",
+			"label": "tgui: build"
+		}
+	]
+}

Build.bat

@@ -0,0 +1,2 @@
+@call tools\build\build
+@pause

README.md

@@ -1,6 +1,6 @@
 ## /tg/station codebase
 
-[![Build Status](https://github.com/tgstation/tgstation/workflows/CI%20Suite/badge.svg)](https://github.com/tgstation/tgstation/actions?query=workflow%3A%22CI+Suite%22) 
+[![Build Status](https://github.com/tgstation/tgstation/workflows/CI%20Suite/badge.svg)](https://github.com/tgstation/tgstation/actions?query=workflow%3A%22CI+Suite%22)
 [![Percentage of issues still open](https://isitmaintained.com/badge/open/tgstation/tgstation.svg)](https://isitmaintained.com/project/tgstation/tgstation "Percentage of issues still open")
 [![Average time to resolve an issue](https://isitmaintained.com/badge/resolution/tgstation/tgstation.svg)](https://isitmaintained.com/project/tgstation/tgstation "Average time to resolve an issue")
 ![Coverage](https://img.shields.io/badge/coverage---3%25-red.svg)

dependencies.sh

@@ -15,6 +15,7 @@ export RUST_G_VERSION=0.4.7
 
 #node version
 export NODE_VERSION=12
+export NODE_VERSION_PRECISE=12.20.0
 
 # SpacemanDMM git tag
 export SPACEMAN_DMM_VERSION=suite-1.6

tgui/.gitattributes

@@ -18,4 +18,5 @@ bin/tgui text eol=lf
 ## Treat bundles as binary and ignore them during conflicts
 *.bundle.* binary merge=tgui-merge-bundle
 *.chunk.* binary merge=tgui-merge-bundle
-.yarn/releases/* binary
+.yarn/releases/**/* binary
+.yarn/plugins/**/* binary

tgui/.gitignore

@@ -13,11 +13,9 @@ package-lock.json
 
 ## Build artifacts
 /public/.tmp/**/*
-/public/**/*.hot-update.*
-/public/**/*.map
+/public/**/*
+!/public/*.html
 
 ## Previously ignored locations that are kept to avoid confusing git
 ## while transitioning to a new project structure.
-/packages/tgui/public/.tmp/**/*
-/packages/tgui/public/**/*.hot-update.*
-/packages/tgui/public/**/*.map
+/packages/tgui/public/**

tgui/.yarnrc.yml

@@ -1 +1,22 @@
-yarnPath: .yarn/releases/yarn-2.3.3.cjs
+logFilters:
+  ## MUST_BUILD
+  ## It's an opencollective advert in postinstall that is not even displayed
+  ## correctly in Yarn, who the fuck cares.
+  - code: YN0007
+    level: discard
+  ## FETCH_NOT_CACHED
+  ## Too many cache misses on first install.
+  - code: YN0013
+    level: discard
+  ## INCOMPATIBLE_OS
+  ## fsevents are not supposed to build, they're macOS specific.
+  - code: YN0062
+    level: discard
+
+plugins:
+  - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.cjs
+    spec: "@yarnpkg/plugin-interactive-tools"
+  - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
+    spec: "@yarnpkg/plugin-workspace-tools"
+
+yarnPath: .yarn/releases/yarn.cjs

tgui/bin/tgui

@@ -26,9 +26,14 @@ fi
 base_dir="$(dirname "$(tgui-realpath "${0}")")/.."
 base_dir="$(tgui-realpath "${base_dir}")"
 
+## Make use of nvm if it exists
+if [[ -e "${HOME}/.nvm/nvm.sh" ]]; then
+  source "${HOME}/.nvm/nvm.sh"
+fi
+
 ## Fall back to running Yarn from the repo
 if ! hash yarn 2>/dev/null; then
-  alias yarn="node '${base_dir}/.yarn/releases/yarn-2.3.3.cjs'"
+  alias yarn="node '${base_dir}/.yarn/releases/yarn.cjs'"
 fi
 
 
@@ -66,6 +71,8 @@ task-clean() {
   ## Build artifacts
   rm -rf public/.tmp
   rm -f public/*.map
+  rm -f public/*.chunk.*
+  rm -f public/*.bundle.*
   rm -f public/*.hot-update.*
   ## Yarn artifacts
   rm -rf .yarn/cache
@@ -78,19 +85,6 @@ task-clean() {
   rm -f **/package-lock.json
 }
 
-## Validates current build against the build stored in git
-task-validate-build() {
-  cd "${base_dir}"
-  local diff
-  diff="$(git diff public/*)"
-  if [[ -n ${diff} ]]; then
-    echo "Error: our build differs from the build committed into git."
-    echo "Please rebuild tgui."
-    exit 1
-  fi
-  echo "tgui: build is ok"
-}
-
 ## Installs merge drivers and git hooks
 task-install-git-hooks() {
   cd "${base_dir}"
@@ -133,16 +127,6 @@ if [[ ${1} == "--install-git-hooks" ]]; then
   exit 0
 fi
 
-## Continuous integration scenario
-if [[ ${1} == "--ci" ]]; then
-  task-clean
-  task-install
-  task-eslint
-  task-webpack --mode=production
-  task-validate-build
-  exit 0
-fi
-
 if [[ ${1} == "--clean" ]]; then
   task-clean
   exit 0
@@ -184,6 +168,13 @@ if [[ ${1} == '--analyze' ]]; then
 fi
 
 ## Make a production webpack build
+if [[ ${1} == '--build' ]]; then
+  task-install
+  task-webpack --mode=production
+  exit 0
+fi
+
+## Make a production webpack build + Run eslint
 if [[ -z ${1} ]]; then
   task-install
   task-eslint

tgui/bin/tgui.ps1

@@ -15,7 +15,7 @@ Set-Location $basedir
 ## --------------------------------------------------------
 
 function yarn {
-  node.exe ".yarn\releases\yarn-2.3.3.cjs" @Args
+  node.exe ".yarn\releases\yarn.cjs" @Args
 }
 
 function Remove-Quiet {

tgui/yarn.lock

@@ -4090,7 +4090,7 @@ fsevents@^1.2.7:
 
 "fsevents@patch:fsevents@^1.2.7#builtin<compat/fsevents>":
   version: 1.2.13
-  resolution: "fsevents@patch:fsevents@npm%3A1.2.13#builtin<compat/fsevents>::version=1.2.13&hash=127e8e"
+  resolution: "fsevents@patch:fsevents@npm%3A1.2.13#builtin<compat/fsevents>::version=1.2.13&hash=11e9ea"
   dependencies:
     bindings: ^1.5.0
     nan: ^2.12.1
@@ -4100,7 +4100,7 @@ fsevents@^1.2.7:
 
 "fsevents@patch:fsevents@~2.1.2#builtin<compat/fsevents>":
   version: 2.1.3
-  resolution: "fsevents@patch:fsevents@npm%3A2.1.3#builtin<compat/fsevents>::version=2.1.3&hash=127e8e"
+  resolution: "fsevents@patch:fsevents@npm%3A2.1.3#builtin<compat/fsevents>::version=2.1.3&hash=11e9ea"
   dependencies:
     node-gyp: latest
   checksum: e2b8c379340e21a786d32c653854c8876f94eb1202dd5be378fd42c062bc123aab5051c32bf0011865257c85982c41ded203f9fe8c9f9c8f8c84dc4672abc0e0

tools/LinuxOneShot/SetupProgram/PreCompile.sh

@@ -34,12 +34,12 @@ if ! { [ -x "$has_git" ] && [ -x "$has_grep" ] && [ -f "/usr/lib/i386-linux-gnu/
 	if ! [ -x "$has_sudo" ]; then
 		dpkg --add-architecture i386
 		apt-get update
-		apt-get install -y git libssl-dev:i386 grep mysql-client
+		apt-get install -y git libssl-dev:i386 grep mysql-client lib32z1 pkg-config libssl-dev
 		rm -rf /var/lib/apt/lists/*
 	else
 		sudo dpkg --add-architecture i386
 		sudo apt-get update
-		sudo apt-get install -y git libssl-dev:i386 grep mysql-client
+		sudo apt-get install -y git libssl-dev:i386 grep mysql-client lib32z1 pkg-config libssl-dev
 		sudo rm -rf /var/lib/apt/lists/*
 	fi
 fi
@@ -58,10 +58,17 @@ fi
 echo "Deploying rust-g..."
 cd rust-g
 git checkout "$RUST_G_VERSION"
-~/.cargo/bin/cargo build --release
-mv target/release/librust_g.so "$1/rust_g"
+env PKG_CONFIG_ALLOW_CROSS=1 ~/.cargo/bin/cargo build --release --target=i686-unknown-linux-gnu
+mv target/i686-unknown-linux-gnu/release/librust_g.so "$1/librust_g.so"
 cd ..
 
+# compile tgui
+echo "Compiling tgui..."
+cd "$1"
+chmod +x tools/bootstrap/node  # Workaround for https://github.com/tgstation/tgstation-server/issues/1167
+env TG_BOOTSTRAP_CACHE="$original_dir" TG_BOOTSTRAP_NODE_LINUX=1 TG_BUILD_TGS_MODE=1 tools/bootstrap/node tools/build/build.js
+cd "$original_dir"
+
 if [ ! -d "../GameStaticFiles/config" ]; then
 	echo "Creating initial config..."
 	cp -r "$1/config" "../GameStaticFiles/config"
@@ -76,3 +83,10 @@ if [ "$DATABASE_EXISTS" != "ss13_db" ]; then
     mysql -u root --password=$MYSQL_ROOT_PASSWORD -h mariadb -P 3306 ss13_db < "$1/$TGS_PREFIXED_SCHEMA_FILE"
     mysql -u root --password=$MYSQL_ROOT_PASSWORD -h mariadb -P 3306 ss13_db -e "INSERT INTO \`SS13_schema_revision\` (\`major\`, \`minor\`) VALUES ($TGS_SCHEMA_MAJOR_VERSION, $TGS_SCHEMA_MINOR_VERSION)"
 fi
+
+# compile tgui
+echo "Compiling tgui..."
+cd "$1"
+chmod +x tools/bootstrap/node  # Workaround for https://github.com/tgstation/tgstation-server/issues/1167
+chmod +x tgui/bin/tgui
+env TG_BOOTSTRAP_CACHE="$original_dir" TG_BOOTSTRAP_NODE_LINUX=1 TG_BUILD_TGS_MODE=1 tools/bootstrap/node tools/build/build.js

tools/LinuxOneShot/SetupProgram/SetupProgram.csproj

@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Tgstation.Server.Client" Version="6.1.0" />
+    <PackageReference Include="Tgstation.Server.Client" Version="8.4.0" />
   </ItemGroup>
 
 </Project>

tools/LinuxOneShot/TGS_Config/appsettings.Production.json

@@ -18,12 +18,5 @@
 	  "Enable": true,
 	  "AllowAnyOrigin": false,
 	  "AllowedOrigins": null
-	},
-	"Kestrel": {
-	  "EndPoints": {
-		"Http": {
-		  "Url": "http://0.0.0.0:80"
-		}
-	  }
 	}
   }