From 49ba1627de0fef9f89d94d46ca1ba832518c5728 Mon Sep 17 00:00:00 2001 From: Max Milton Date: Sun, 5 Jun 2022 08:13:06 +1000 Subject: [PATCH] bug: Move DB table stats feature behind config flag (temp workaround for #158) (#234) --- packages/trackx-api/src/routes/dash/stats.ts | 15 ++++++-- packages/trackx-api/src/types.ts | 12 +++++- packages/trackx-api/src/utils.ts | 20 ++++++++++ packages/trackx-api/trackx.config.js | 2 + packages/trackx-api/trackx.config.js.template | 2 + packages/trackx-cli/src/utils.ts | 7 +++- packages/trackx-dash/src/pages/stats.tsx | 38 +++++++++++-------- 7 files changed, 74 insertions(+), 22 deletions(-) diff --git a/packages/trackx-api/src/routes/dash/stats.ts b/packages/trackx-api/src/routes/dash/stats.ts index 87e840b9..2ba24817 100644 --- a/packages/trackx-api/src/routes/dash/stats.ts +++ b/packages/trackx-api/src/routes/dash/stats.ts @@ -13,6 +13,7 @@ import { AppError, config, humanFileSize, + humanizeTime, logger, sessions, Status, @@ -280,8 +281,7 @@ export const get: Middleware = async (req, res, next) => { throw new AppError('Unexpected param', Status.BAD_REQUEST); } - const [tableSizes, dbFileSize, dbWalFileSize] = await Promise.all([ - getTableSizes(), + const [dbFileSize, dbWalFileSize] = await Promise.all([ fs.promises.stat(config.DB_PATH).then(humanizeSize), fs.promises.stat(`${config.DB_PATH}-wal`).then(humanizeSize), ]); @@ -291,7 +291,16 @@ export const get: Middleware = async (req, res, next) => { data.api_uptime = Math.floor(process.uptime()); data.db_size = dbFileSize; data.dbwal_size = dbWalFileSize; - data.db_tables = tableSizes; + // data.db_tables = tableSizes; + + // FIXME: Generating DB table stats is extremely slow on systems with slow disks + // ↳ https://github.com/maxmilton/trackx/issues/158 + if (config.ENABLE_DB_TABLE_STATS) { + const t0 = performance.now(); + data.db_tables = await getTableSizes(); + const t1 = performance.now(); + logger.info(`Generated DB table stats in ${humanizeTime(t1 - t0)}`); + } send(res, Status.OK, data); } catch (error) { diff --git a/packages/trackx-api/src/types.ts b/packages/trackx-api/src/types.ts index bf529917..c2d667ff 100644 --- a/packages/trackx-api/src/types.ts +++ b/packages/trackx-api/src/types.ts @@ -147,6 +147,15 @@ export interface TrackXAPIConfig { * token expires, the user must login again. */ readonly SESSION_TTL: number; + + /** + * Show database table size statistics on the dash `/stats` page. + * + * @remarks Currently has known performance issues. Enabling this feature may + * be useful for debugging and observability but should be disabled in + * production to prevent a potential denial-of-service attack vector. + */ + readonly ENABLE_DB_TABLE_STATS?: boolean; } export type ReqBodyData = Record; @@ -325,7 +334,8 @@ export interface Stats { dash_session_c: number; db_size: string; dbwal_size: string; - db_tables: StatsDBTableInfo[]; + // Optional because it's behind the config flag "ENABLE_DB_TABLE_STATS" + db_tables?: StatsDBTableInfo[]; } export interface Logs { diff --git a/packages/trackx-api/src/utils.ts b/packages/trackx-api/src/utils.ts index 37119ec6..448d021e 100644 --- a/packages/trackx-api/src/utils.ts +++ b/packages/trackx-api/src/utils.ts @@ -190,6 +190,26 @@ export function generateSalt(rounds: number): string { .slice(0, rounds); } +export function humanizeTime(ms: number): string { + const periods = { + d: Math.floor(ms / (1000 * 60 * 60 * 24)), + h: Math.floor((ms % (1000 * 60 * 60)) / 24), + m: Math.floor((ms % (1000 * 60)) / 60), + s: Math.floor((ms % 1000) / 60), + ms: Math.floor(ms % 1000), + }; + const keep = []; + let key: keyof typeof periods; + + for (key in periods) { + if (periods[key] > 0) { + keep.push(`${periods[key]}${key}`); + } + } + + return keep.join(' '); +} + export function humanFileSize(bytes: number): string { if (bytes < 1024) return `${bytes} B`; let b = bytes; diff --git a/packages/trackx-api/trackx.config.js b/packages/trackx-api/trackx.config.js index d092b518..5808fec0 100644 --- a/packages/trackx-api/trackx.config.js +++ b/packages/trackx-api/trackx.config.js @@ -35,4 +35,6 @@ module.exports = { NET_TIMEOUT: 30_000, // 30 seconds SCHEDULED_JOB_INTERVAL: 21_600_000, // 6 hours; 6h * 60m * 60s * 1000ms SESSION_TTL: 2_400_000, // 40 minutes; 40m * 60s * 1000ms + + ENABLE_DB_TABLE_STATS: true, }; diff --git a/packages/trackx-api/trackx.config.js.template b/packages/trackx-api/trackx.config.js.template index 0424c94d..55d4064b 100644 --- a/packages/trackx-api/trackx.config.js.template +++ b/packages/trackx-api/trackx.config.js.template @@ -22,4 +22,6 @@ module.exports = { NET_TIMEOUT: 30000, // 30 seconds SCHEDULED_JOB_INTERVAL: 21600000, // 6 hours SESSION_TTL: 2400000, // 40 minutes + + ENABLE_DB_TABLE_STATS: false, }; diff --git a/packages/trackx-cli/src/utils.ts b/packages/trackx-cli/src/utils.ts index a2c6c65e..8cfe4804 100644 --- a/packages/trackx-cli/src/utils.ts +++ b/packages/trackx-cli/src/utils.ts @@ -53,6 +53,8 @@ const CONFIG_SCHEMA = [ ['NET_TIMEOUT', ['Number']], ['SCHEDULED_JOB_INTERVAL', ['Number']], ['SESSION_TTL', ['Number']], + + ['ENABLE_DB_TABLE_STATS', ['Boolean', 'Undefined']], ] as const; const configExpectedKeys: string[] = CONFIG_SCHEMA.map((item) => item[0]); @@ -84,8 +86,9 @@ function validateConfig( } for (const [key, types] of CONFIG_SCHEMA) { - if (!(key in rawConfig)) { - errors.push(new ReferenceError(`Config missing "${key}" key`)); + // @ts-expect-error - FIXME: "types" type + if (!(key in rawConfig) && !types.includes('Undefined')) { + errors.push(new ReferenceError(`Config missing required "${key}" key`)); } const valueType = toStr diff --git a/packages/trackx-dash/src/pages/stats.tsx b/packages/trackx-dash/src/pages/stats.tsx index 67ed4c2f..6f96b874 100644 --- a/packages/trackx-dash/src/pages/stats.tsx +++ b/packages/trackx-dash/src/pages/stats.tsx @@ -139,28 +139,34 @@ const StatsPage: Component = () => {

Database

-

+

{data.db_size} + {data.dbwal_size} (wal)

-

Tables

- -
- - - {([name, size, percent]) => ( - - - )} - -
- - -
-
+ {/* FIXME: Generating DB table stats is extremely slow on systems + with slow disks -- https://github.com/maxmilton/trackx/issues/158 */} + {data.db_tables && ( + <> +

Tables

+ +
+ + + {([name, size, percent]) => ( + + + )} + +
+ + +
+
+ + )}
)}