Ezdb (tgstation#17156)

Co-authored-by: TiviPlus <[email protected]>

.github/guides/EZDB.md

@@ -0,0 +1,12 @@
+# Quickly setting up a development database with ezdb
+While you do not need a database to code for tgmc, it is a prerequisite to many important features, especially on the admin side. Thus, if you are working in any code that benefits from it, it can be helpful to have one handy.
+
+**ezdb** is a tool for quickly setting up an isolated development database. It will manage downloading MariaDB, creating the database, setting it up, and updating it when the code evolves. It is not recommended for use in production servers, but is perfect for quick development.
+
+To run ezdb, go to `tools/ezdb`, and double-click on ezdb.bat. This will set up the database on port 1338, but you can configure this with `--port`. When it is done, you should be able to launch tgstation as normal and have database access. This runs on the same Python bootstrapper as things like the map merge tool, which can sometimes be flaky.
+
+If you wish to delete the ezdb database, delete the `db` folder as well as `config/ezdb.txt`.
+
+To update ezdb, run the script again. This will both look for any updates in the database changelog, as well as update your schema revision.
+
+Contact Mothblocks if you face any issues in this process.

.gitignore

@@ -57,6 +57,10 @@ tools/hyperscale_converter/output/*
 # From /tools/define_sanity/check.py - potential output file that we load onto the user's machine that we don't want to have committed.
 define_sanity_output.txt
 
+# ezdb
+/db/
+/config/ezdb.txt
+
 # Running OpenDream locally
 tgmc.json
 rust_g64.dll

CONTRIBUTING.md

@@ -19,6 +19,9 @@
 10. [Rust in TGMC](#rust)
 11. [A word on Git](#a-word-on-git)
 
+#### Misc
+- [Quickly setting up a development database with ezdb](./guides/EZDB.md)
+
 ## Reporting Issues
 
 See [this page](https://github.com/tgstation/TerraGov-Marine-Corps/issues/new?template=bug_report.md) for a guide and format to issue reports.

code/controllers/configuration/configuration.dm

@@ -82,6 +82,8 @@
 				for(var/J in legacy_configs)
 					LoadEntries(J)
 				break
+	if (fexists("[directory]/ezdb.txt"))
+		LoadEntries("ezdb.txt")
 	loadmaplist(CONFIG_GROUND_MAPS_FILE, GROUND_MAP)
 	loadmaplist(CONFIG_SHIP_MAPS_FILE, SHIP_MAP)
 	LoadMOTD()

code/controllers/configuration/entries/dbconfig.dm

@@ -47,3 +47,8 @@
 /datum/config_entry/number/bsql_thread_limit
 	config_entry_value = 50
 	min_val = 1
+
+/// The exe for mariadbd.exe.
+/// Shouldn't really be set on production servers, primarily for EZDB.
+/datum/config_entry/string/db_daemon
+	protection = CONFIG_ENTRY_LOCKED | CONFIG_ENTRY_HIDDEN

code/controllers/subsystem/dbcore.dm

@@ -15,6 +15,8 @@ SUBSYSTEM_DEF(dbcore)
 
 	var/connection  // Arbitrary handle returned from rust_g.
 
+	var/db_daemon_started = FALSE
+
 /datum/controller/subsystem/dbcore/Initialize()
 	//We send warnings to the admins during subsystem init, as the clients will be New'd and messages
 	//will queue properly with goonchat
@@ -50,6 +52,7 @@ SUBSYSTEM_DEF(dbcore)
 		qdel(query_round_shutdown)
 	if(IsConnected())
 		Disconnect()
+	stop_db_daemon()
 
 //nu
 /datum/controller/subsystem/dbcore/can_vv_get(var_name)
@@ -74,6 +77,8 @@ SUBSYSTEM_DEF(dbcore)
 	if(!CONFIG_GET(flag/sql_enabled))
 		return FALSE
 
+	start_db_daemon()
+
 	var/user = CONFIG_GET(string/feedback_login)
 	var/pass = CONFIG_GET(string/feedback_password)
 	var/db = CONFIG_GET(string/feedback_database)
@@ -275,6 +280,47 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table
 		. = Query.Execute(async)
 	qdel(Query)
 
+/datum/controller/subsystem/dbcore/proc/start_db_daemon()
+	set waitfor = FALSE
+
+	if (db_daemon_started)
+		return
+
+	db_daemon_started = TRUE
+
+	var/daemon = CONFIG_GET(string/db_daemon)
+	if (!daemon)
+		return
+
+	ASSERT(fexists(daemon), "Configured db_daemon doesn't exist")
+
+	var/list/result = world.shelleo("echo \"Starting ezdb daemon, do not close this window\" && [daemon]")
+	var/result_code = result[1]
+	if (!result_code || result_code == 1)
+		return
+
+	stack_trace("Failed to start DB daemon: [result_code]\n[result[3]]")
+
+/datum/controller/subsystem/dbcore/proc/stop_db_daemon()
+	set waitfor = FALSE
+
+	if (!db_daemon_started)
+		return
+
+	db_daemon_started = FALSE
+
+	var/daemon = CONFIG_GET(string/db_daemon)
+	if (!daemon)
+		return
+
+	switch (world.system_type)
+		if (MS_WINDOWS)
+			var/list/result = world.shelleo("Get-Process | ? { $_.Path -eq '[daemon]' } | Stop-Process")
+			ASSERT(result[1], "Failed to stop DB daemon: [result[3]]")
+		if (UNIX)
+			var/list/result = world.shelleo("kill $(pgrep -f '[daemon]')")
+			ASSERT(result[1], "Failed to stop DB daemon: [result[3]]")
+
 /datum/db_query
 	// Inputs
 	var/connection

tools/ezdb/__main__.py

@@ -0,0 +1,15 @@
+import argparse
+from .steps import STEPS
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--port", type = int, default = 1338)
+
+args = parser.parse_args()
+
+for step in STEPS:
+    if not step.should_run():
+        continue
+
+    step.run(args)
+
+print("Done!")

tools/ezdb/ezdb.bat

@@ -0,0 +1,2 @@
+@call "%~dp0\..\bootstrap\python" -m ezdb %*
+@pause

tools/ezdb/ezdb/changes.py

@@ -0,0 +1,31 @@
+import re
+from dataclasses import dataclass
+from .paths import get_changelog_path
+
+REGEX_CHANGE = r"-+\s*Version (?P<major>[0-9]+)\.(?P<minor>[0-9]+), .+?\`\`\`sql\s*(?P<sql>.+?)\s*\`\`\`.*?-{5}"
+
+@dataclass
+class Change:
+    major_version: int
+    minor_version: int
+    sql: str
+
+def get_changes() -> list[Change]:
+    with open(get_changelog_path(), "r") as file:
+        changelog = file.read()
+        changes = []
+
+        for change_match in re.finditer(REGEX_CHANGE, changelog, re.MULTILINE | re.DOTALL):
+            changes.append(Change(
+                int(change_match.group("major")),
+                int(change_match.group("minor")),
+                change_match.group("sql")
+            ))
+
+        changes.sort(key = lambda change: (change.major_version, change.minor_version), reverse = True)
+
+        return changes
+
+def get_current_version():
+    changes = get_changes()
+    return (changes[0].major_version, changes[0].minor_version)

tools/ezdb/ezdb/config.py

@@ -0,0 +1,22 @@
+from .paths import get_config_path
+from typing import Optional
+
+def read_config() -> Optional[dict[str, str]]:
+    config_path = get_config_path()
+    if not config_path.exists():
+        return None
+
+    with config_path.open('r') as file:
+        lines = file.readlines()
+        entries = {}
+
+        for line in lines:
+            if line.startswith("#"):
+                continue
+            if " " not in line:
+                continue
+
+            key, value = line.split(" ", 1)
+            entries[key.strip()] = value.strip()
+
+        return entries

tools/ezdb/ezdb/mysql.py

@@ -0,0 +1,68 @@
+import atexit
+import mysql.connector
+import subprocess
+from contextlib import closing
+from .config import read_config
+from .paths import get_mariadb_client_path, get_mariadb_daemon_path
+
+def open_connection():
+    config = read_config()
+    assert config["FEEDBACK_PASSWORD"] is not None, "No password found in config file"
+
+    connection = mysql.connector.connect(
+        user = config["FEEDBACK_LOGIN"],
+        password = config["FEEDBACK_PASSWORD"],
+        port = int(config["PORT"]),
+        raise_on_warnings = True,
+    )
+
+    connection.autocommit = True
+
+    return closing(connection)
+
+# We use custom things like delimiters, so we can't use the built-in cursor.execute
+def execute_sql(sql: str):
+    config = read_config()
+    assert config is not None, "No config file found"
+    assert config["FEEDBACK_PASSWORD"] is not None, "No password found in config file"
+
+    subprocess.run(
+        [
+            str(get_mariadb_client_path()),
+            "-u",
+            "root",
+            "-p" + config["FEEDBACK_PASSWORD"],
+            "--port",
+            config["PORT"],
+            "--database",
+            config["FEEDBACK_DATABASE"],
+        ],
+        input = sql,
+        encoding = "utf-8",
+        check = True,
+        stderr = subprocess.STDOUT,
+    )
+
+def insert_new_schema_query(major_version: int, minor_version: int):
+    return f"INSERT INTO `schema_revision` (`major`, `minor`) VALUES ({major_version}, {minor_version})"
+
+process = None
+def start_daemon():
+    global process
+    if process is not None:
+        return
+
+    print("Starting MariaDB daemon...")
+    config = read_config()
+    assert config is not None, "No config file found"
+
+    process = subprocess.Popen(
+        [
+            str(get_mariadb_daemon_path()),
+            "--port",
+            config["PORT"],
+        ],
+        stderr = subprocess.PIPE,
+    )
+
+    atexit.register(process.kill)

tools/ezdb/ezdb/paths.py

@@ -0,0 +1,34 @@
+import pathlib
+
+def get_root_path():
+    current_path = pathlib.Path(__file__)
+    while current_path.name != 'tools':
+        current_path = current_path.parent
+    return current_path.parent
+
+def get_config_path():
+    return get_root_path() / 'config' / 'ezdb.txt'
+
+def get_db_path():
+    return get_root_path() / 'db'
+
+def get_data_path():
+    return get_db_path() / 'data'
+
+def get_mariadb_bin_path():
+    return get_db_path() / 'bin'
+
+def get_mariadb_client_path():
+    return get_mariadb_bin_path() / 'mariadb.exe'
+
+def get_mariadb_daemon_path():
+    return get_mariadb_bin_path() / 'mariadbd.exe'
+
+def get_mariadb_install_db_path():
+    return get_mariadb_bin_path() / 'mariadb-install-db.exe'
+
+def get_initial_schema_path():
+    return get_root_path() / 'SQL' / 'tgmc-schema.sql'
+
+def get_changelog_path():
+    return get_root_path() / 'SQL' / 'database_changelog.md'

tools/ezdb/steps/__init__.py

@@ -0,0 +1,11 @@
+from .download_mariadb import DownloadMariaDB
+from .install_database import InstallDatabase
+from .install_initial_schema import InstallInitialSchema
+from .update_schema import UpdateSchema
+
+STEPS = [
+    DownloadMariaDB,
+    InstallDatabase,
+    InstallInitialSchema,
+    UpdateSchema,
+]

tools/ezdb/steps/download_mariadb.py

@@ -0,0 +1,50 @@
+import os
+import pathlib
+import tempfile
+import urllib.request
+import zipfile
+from ..ezdb.paths import get_config_path, get_data_path, get_db_path, get_mariadb_bin_path, get_mariadb_daemon_path, get_mariadb_install_db_path
+from .step import Step
+
+# Theoretically, this could use the REST API that MariaDB has to find the URL given a version:
+# https://downloads.mariadb.org/rest-api/mariadb/10.11
+DOWNLOAD_URL = "http://downloads.mariadb.org/rest-api/mariadb/10.11.2/mariadb-10.11.2-winx64.zip"
+FOLDER_NAME = "mariadb-10.11.2-winx64"
+
+temp_extract_path = get_db_path() / "_temp/"
+
+class DownloadMariaDB(Step):
+    @staticmethod
+    def should_run() -> bool:
+        return not get_mariadb_bin_path().exists()
+
+    @staticmethod
+    def run(args):
+        if temp_extract_path.exists():
+            print("Deleting old temporary extract folder")
+            temp_extract_path.rmdir()
+
+        print("Downloading portable MariaDB...")
+
+        # delete = False so we can write to it
+        temporary_file = tempfile.NamedTemporaryFile(delete = False)
+
+        try:
+            urllib.request.urlretrieve(DOWNLOAD_URL, temporary_file.name)
+
+            print("Extracting...")
+            os.makedirs(temp_extract_path, exist_ok = True)
+            with zipfile.ZipFile(temporary_file) as zip_file:
+                for file in zip_file.namelist():
+                    if file.startswith(f"{FOLDER_NAME}/bin/"):
+                        with zip_file.open(file) as source, open(temp_extract_path / pathlib.Path(file).name, "wb") as target:
+                            target.write(source.read())
+
+            print("Moving...")
+
+            temp_extract_path.rename(get_mariadb_bin_path())
+        finally:
+            temporary_file.close()
+
+            if temp_extract_path.exists():
+                temp_extract_path.rmdir()