Adds environment variable overrides for all CLI args (#844)

* adding a new centralized place to define all possible options in various groupings.

* fixing up affected and cache commands

* modified all the commands

* Change files

* get rid unused files

* adding clarifications

change/change-a5b5c59d-4f88-482f-8ff8-9fa13a6d62ed.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "type": "minor",
+      "comment": "Adds environment variables override for all CLI args. Get these from --help",
+      "packageName": "@lage-run/cli",
+      "email": "[email protected]",
+      "dependentChangeType": "patch"
+    }
+  ]
+}

packages/cli/src/commands/addOptions.ts

@@ -0,0 +1,9 @@
+import type { Command, Option } from "commander";
+import { options } from "./options.js";
+
+export function addOptions(category: keyof typeof options, command: Command) {
+  for (const option of Object.values<Option>(options[category])) {
+    command.addOption(option);
+  }
+  return command;
+}

packages/cli/src/commands/affected/index.ts

@@ -1,16 +1,8 @@
 import { Command } from "commander";
-import { addFilterOptions } from "../addFilterOptions.js";
 import { affectedAction } from "./action.js";
+import { addOptions } from "../addOptions.js";
 
 const affectedCommand = new Command("affected");
-
-addFilterOptions(affectedCommand)
-  .action(affectedAction)
-  .option(
-    "--output-format <graph|json|default>",
-    `Generate a report about what packages are affected by the current change (defaults to human readable format) ` +
-      `"graph" will generate a GraphViz .dot file format`
-  )
-  .option("--since <branch>", "Calculate changes since this branch (defaults to origin/master)", "origin/master");
+addOptions("filter", affectedCommand).action(affectedAction);
 
 export { affectedCommand };

packages/cli/src/commands/cache/index.ts

@@ -1,12 +1,11 @@
-import { Command, Option } from "commander";
-import { addLoggerOptions } from "../addLoggerOptions.js";
+import { Command } from "commander";
 import { cacheAction } from "./action.js";
+import { addOptions } from "../addOptions.js";
 
-const cacheCommand = new Command("cache");
+const command = new Command("cache");
 
-addLoggerOptions(cacheCommand)
-  .action(cacheAction)
-  .addOption(new Option("--prune <days>", "Prunes cache older than certain number of <days>").argParser(parseInt).conflicts("--clear"))
-  .option("--clear", "Clears the cache locally");
+addOptions("cache", command);
+addOptions("logger", command);
+command.action(cacheAction);
 
-export { cacheCommand };
+export { command as cacheCommand };

packages/cli/src/commands/exec/index.ts

@@ -1,16 +1,12 @@
 import { Command } from "commander";
 import { execAction } from "./action.js";
-import { addLoggerOptions } from "../addLoggerOptions.js";
-import os from "os";
+import { addOptions } from "../addOptions.js";
 
-const execCommand = new Command("exec");
-execCommand.option(
-  "-n|--node-arg <arg>",
-  "node argument to pass to worker, just a single string to be passed into node like a NODE_OPTIONS setting"
-);
-execCommand.option("-c|--concurrency <number>", "max jobs to run at a time", (v) => parseInt(v), os.cpus().length - 1);
-execCommand.option("-s|--server [host:port]", "lage server host");
-execCommand.option<number>("-t|--timeout <seconds>", "lage server autoshutoff timeout", (v) => parseInt(v), 5 * 60);
-execCommand.option("--tasks [tasks...]", "A list of tasks to run, separated by space e.g. 'build test', used with --server");
-addLoggerOptions(execCommand).action(execAction);
-export { execCommand };
+const command = new Command("exec");
+addOptions("pool", command);
+addOptions("runner", command);
+addOptions("server", command);
+addOptions("logger", command);
+command.action(execAction);
+
+export { command as execCommand };

packages/cli/src/commands/info/index.ts

@@ -1,17 +1,12 @@
 import { Command } from "commander";
 import { infoAction } from "./action.js";
-import { addFilterOptions } from "../addFilterOptions.js";
-import { addLoggerOptions } from "../addLoggerOptions.js";
+import { addOptions } from "../addOptions.js";
 
-const infoCommand = new Command("info");
+const command = new Command("info");
+addOptions("server", command);
+addOptions("runner", command);
+addOptions("logger", command);
+addOptions("filter", command);
+command.action(infoAction);
 
-addFilterOptions(addLoggerOptions(infoCommand));
-infoCommand.description("Display information about a target graph in a workspace.\n" + "It is used by BuildXL to build a pip-graph");
-infoCommand.option("--server [host:port]", "Run targets of type 'worker' on a background service");
-infoCommand.option(
-  "--nodearg|--node-arg <nodeArg>",
-  'arguments to be passed to node (e.g. --nodearg="--max_old_space_size=1234 --heap-prof" - set via "NODE_OPTIONS" environment variable'
-);
-infoCommand.action(infoAction);
-
-export { infoCommand };
+export { command as infoCommand };

packages/cli/src/commands/init/index.ts

@@ -1,9 +1,8 @@
 import { Command } from "commander";
-import { addFilterOptions } from "../addFilterOptions.js";
 import { initAction } from "./action.js";
 
 const initCommand = new Command("init");
 
-addFilterOptions(initCommand).description("Install lage in a workspace and create a config file").action(initAction);
+initCommand.description("Install lage in a workspace and create a config file").action(initAction);
 
 export { initCommand };

packages/cli/src/commands/launchServerInBackground.ts

@@ -3,7 +3,7 @@ import fs from "fs";
 import path from "path";
 import lockfile from "proper-lockfile";
 import execa from "execa";
-import { getBinPaths, getBinScripts } from "../getBinPaths.js";
+import { getBinScripts } from "../getBinPaths.js";
 
 export interface launchServerInBackgroundOptions {
   logger: Logger;
@@ -58,10 +58,8 @@ export async function launchServerInBackground({
       lageServerBinPath,
       "--tasks",
       ...tasks,
-      "--host",
-      host,
-      "--port",
-      `${port}`,
+      "--server",
+      `${host}:${port}`,
       "--timeout",
       `${timeout}`,
       ...args,

packages/cli/src/commands/options.ts

@@ -0,0 +1,87 @@
+import { Option } from "commander";
+
+const isCI = process.env.CI || process.env.TF_BUILD;
+
+const options = {
+  logger: {
+    reporter: new Option("--reporter <reporter...>", "reporter"),
+    grouped: new Option("--grouped", "groups the logs").default(false),
+    progress: new Option("--progress").conflicts(["reporter", "grouped", "verbose"]).default(!isCI),
+    logLevel: new Option("--log-level <level>", "log level").choices(["info", "warn", "error", "verbose", "silly"]).conflicts("verbose"),
+    logFile: new Option("--log-file <file>", "when used with --reporter vfl, writes verbose, ungrouped logs to the specified file"),
+    verbose: new Option("--verbose", "verbose output").default(false),
+  },
+  pool: {
+    concurrency: new Option("-c|--concurrency <number>", "max jobs to run at a time").argParser((v) => parseInt(v)),
+    continue: new Option("--continue", "continue running even after encountering an error for one of the targets"),
+    maxWorkersPerTask: new Option(
+      "--max-workers-per-task <maxWorkersPerTarget...>",
+      "set max worker per task, e.g. --max-workers-per-task build=2 test=4"
+    ).default([]),
+  },
+  runner: {
+    nodeArg: new Option(
+      "-n|--node-arg <arg>",
+      'node arguments as a string to be passed into node like a NODE_OPTIONS setting, (e.g. --nodearg="--max_old_space_size=1234 --heap-prof")'
+    ),
+  },
+  run: {
+    cache: new Option("--no-cache", "disables the cache"),
+    resetCache: new Option("--reset-cache", "resets the cache, filling it after a run"),
+    skipLocalCache: new Option("--skip-local-cache", "skips caching locally (defaults to true in CI environments)").default(isCI),
+    profile: new Option("--profile [profile]", "writes a run profile into a file that can be processed by Chromium devtool"),
+    continue: new Option("--continue", "continues the run even on error"),
+    allowNoTargetRuns: new Option("--allow-no-target-runs"),
+    watch: new Option("--watch", "runs in watch mode"),
+  },
+  server: {
+    server: new Option("--server [host:port]", "Run targets of type 'worker' on a background service"),
+    tasks: new Option("--tasks <tasks...>", "A list of tasks to run, separated by space e.g. 'build test'"),
+    timeout: new Option("-t|--timeout <seconds>", "lage server autoshutoff timeout").default(5 * 60).argParser((v) => parseInt(v)),
+  },
+  filter: {
+    scope: new Option(
+      "--scope <scope...>",
+      "scopes the run to a subset of packages (by default, includes the dependencies and dependents as well)"
+    ),
+    noDeps: new Option("--no-deps|--no-dependents", "disables running any dependents of the scoped packages"),
+    includeDependencies: new Option(
+      "--include-dependencies|--dependencies",
+      'adds the scoped packages dependencies as the "entry points" for the target graph run'
+    ),
+    to: new Option("--to <scope...>", "runs up to a package (shorthand for --scope=<scope...> --no-dependents)"),
+    since: new Option("--since <since>", "only runs packages that have changed since the given commit, tag, or branch"),
+    ignore: new Option(
+      "--ignore <ignore...>",
+      "ignores files when calculating the scope with `--since` in addition to the files specified in lage.config"
+    ).default([]),
+  },
+  affected: {
+    outputFormat: new Option(
+      "--output-format <graph|json|default>",
+      `Generate a report about what packages are affected by the current change (defaults to human readable format) ` +
+        `"graph" will generate a GraphViz .dot file format`
+    ),
+  },
+  cache: {
+    prune: new Option("--prune <days>", "Prunes cache older than certain number of <days>").argParser(parseInt).conflicts("--clear"),
+    clear: new Option("--clear", "Clears the cache locally"),
+  },
+} as const;
+
+const optionsWithEnv = addEnvOptions(options);
+
+function addEnvOptions(opts: typeof options) {
+  for (const key in opts) {
+    for (const [name, option] of Object.entries<Option>(opts[key])) {
+      // convert the camel cased name to uppercase with underscores
+      const upperCaseSnakeKey = key.replace(/([A-Z])/g, "_$1").toUpperCase();
+      const upperCaseSnakeName = name.replace(/([A-Z])/g, "_$1").toUpperCase();
+      option.env(`LAGE_${upperCaseSnakeKey}_${upperCaseSnakeName}`);
+    }
+  }
+
+  return opts;
+}
+
+export { optionsWithEnv as options };

packages/cli/src/commands/run/index.ts

@@ -1,28 +1,17 @@
 import { Command } from "commander";
 import { action } from "./action.js";
-import { addLoggerOptions } from "../addLoggerOptions.js";
-import { isRunningFromCI } from "../isRunningFromCI.js";
-import { addFilterOptions } from "../addFilterOptions.js";
+import { addOptions } from "../addOptions.js";
 
-const runCommand = new Command("run");
+const command = new Command("run");
 
-addFilterOptions(addLoggerOptions(runCommand))
+addOptions("filter", command);
+addOptions("logger", command);
+addOptions("pool", command);
+addOptions("runner", command);
+addOptions("run", command);
+
+command
   .action(action)
-  .option("-c, --concurrency <n>", "concurrency", (value) => parseInt(value, 10))
-  .option("--max-workers-per-task <maxWorkersPerTarget...>", "set max worker per task, e.g. --max-workers-per-task build=2 test=4", [])
-
-  // Run Command Options
-  .option("--no-cache", "disables the cache")
-  .option("--reset-cache", "resets the cache, filling it after a run")
-  .option("--skip-local-cache", "skips caching locally (defaults to true in CI environments)", isRunningFromCI)
-  .option("--profile [profile]", "writes a run profile into a file that can be processed by Chromium devtool")
-  .option(
-    "--nodearg|--node-arg <nodeArg>",
-    'arguments to be passed to node (e.g. --nodearg="--max_old_space_size=1234 --heap-prof" - set via "NODE_OPTIONS" environment variable'
-  )
-  .option("--continue", "continues the run even on error")
-  .option("--allow-no-target-runs")
-  .option("--watch", "runs in watch mode")
   .allowUnknownOption(true)
   .addHelpCommand("[run] command1 [command2...commandN] [options]", "run commands")
   .addHelpText(
@@ -92,4 +81,4 @@ Ignoring files when calculating the scope with --since in addition to files spec
 `
   );
 
-export { runCommand };
+export { command as runCommand };

packages/cli/src/commands/server/action.ts

@@ -3,18 +3,20 @@ import type { ReporterInitOptions } from "../../types/ReporterInitOptions.js";
 import { initializeReporters } from "../initializeReporters.js";
 import { createLageService } from "./lageService.js";
 import { createServer } from "@lage-run/rpc";
+import { parseServerOption } from "../parseServerOption.js";
 
 interface WorkerOptions extends ReporterInitOptions {
   nodeArg?: string[];
-  port?: number;
-  host?: string;
+  server?: string;
   timeout?: number;
   shutdown: boolean;
   tasks: string[];
 }
 
 export async function serverAction(options: WorkerOptions) {
-  const { port = 5332, host = "localhost", timeout = 1, tasks } = options;
+  const { server = "localhost:5332", timeout = 1, tasks } = options;
+
+  const { host, port } = parseServerOption(server);
 
   const logger = createLogger();
   options.logLevel = options.logLevel ?? "info";
@@ -37,9 +39,9 @@ export async function serverAction(options: WorkerOptions) {
     concurrency: options.concurrency,
     tasks,
   });
-  const server = await createServer(lageService, abortController);
+  const lageServer = await createServer(lageService, abortController);
 
-  await server.listen({ host, port });
+  await lageServer.listen({ host, port });
   logger.info(`Server listening on http://${host}:${port}, timeout in ${timeout} seconds`);
 }
 

packages/cli/src/commands/server/index.ts

@@ -1,19 +1,14 @@
-import os from "os";
 import { Command } from "commander";
 import { serverAction } from "./action.js";
-import { addLoggerOptions } from "../addLoggerOptions.js";
+import { addOptions } from "../addOptions.js";
 
-const serverCommand = new Command("server");
-serverCommand.option(
-  "-n|--node-arg <arg>",
-  "node argument to pass to worker, just a single string to be passed into node like a NODE_OPTIONS setting"
-);
-serverCommand.option("-c|--concurrency <number>", "max jobs to run at a time", (v) => parseInt(v));
-serverCommand.option("-h|--host <host>", "lage server host", "localhost");
-serverCommand.option<number>("-p|--port <port>", "lage worker server port", (v) => parseInt(v), 5332);
-serverCommand.option<number>("-t|--timeout <seconds>", "lage server autoshutoff timeout", (v) => parseInt(v), 5 * 60);
-serverCommand.option("--tasks <tasks...>", "A list of tasks to run, separated by space e.g. 'build test'");
+const command = new Command("server");
 
-addLoggerOptions(serverCommand).action(serverAction);
+command.action(serverAction);
+addOptions("server", command);
+addOptions("pool", command);
+addOptions("runner", command);
+addOptions("logger", command);
+command.action(serverAction);
 
-export { serverCommand };
+export { command as serverCommand };