feature/desktop cli log formatter (#890)

* Install winston * Set up a logger * Update depcation warnings
whitphx · May 2, 2024 · 502d05d · 502d05d
1 parent a7f0566
commit 502d05d
Show file tree

Hide file tree

Showing 7 changed files with 231 additions and 66 deletions.
diff --git a/packages/desktop/bin-src/dump_artifacts/config.ts b/packages/desktop/bin-src/dump_artifacts/config.ts
@@ -1,5 +1,6 @@
 import path from "node:path";
 import * as s from "superstruct";
+import { deprecationWarning } from "./logger";
 
 interface ReadConfigOptions {
   pathResolutionRoot: string;
@@ -40,24 +41,24 @@ function readFilesAndEntrypoint(options: ReadConfigOptions) {
   let files = packageJsonStliteDesktopField?.files;
   let entrypoint = packageJsonStliteDesktopField?.entrypoint;
   if (files == null || entrypoint == null) {
-    console.warn(
-      "`stlite.desktop.files` and `stlite.desktop.entrypoint` are not found in `package.json`. " +
-        "Read the `appHomeDirSource` argument as the app directory. " +
-        "This behavior will be deprecated in the future."
+    deprecationWarning(
+      "Missing `stlite.desktop.files` and `stlite.desktop.entrypoint` in `package.json`. " +
+        "Falling back to the `appHomeDirSource` argument to determine the app directory. " +
+        "Please update your `package.json` as this fallback is deprecated and will be removed in a future release."
     );
     const appHomeDirSource = appHomeDirSourceFallback;
     if (typeof appHomeDirSource !== "string") {
       throw new Error(
-        "The `appHomeDirSource` argument is required when `stlite.desktop.files` and `stlite.desktop.entrypoint` are not found in the package.json.\n" +
+        "The `appHomeDirSource` argument is required when `stlite.desktop.files` and `stlite.desktop.entrypoint` are not found in the package.json. " +
           "Note that `appHomeDirSource` is deprecated and will be removed in the future, so please specify `stlite.desktop.files` and `stlite.desktop.entrypoint` in the package.json."
       );
     }
     files = [appHomeDirSource];
     entrypoint = `./${appHomeDirSource}/streamlit_app.py`;
   } else {
     if (appHomeDirSourceFallback != null) {
-      console.warn(
-        "[Deprecated] `appHomeDirSource` is ignored because `stlite.desktop.files` is found in `package.json`."
+      deprecationWarning(
+        "The `appHomeDirSource` argument is deprecated and has been ignored because `stlite.desktop.files` is specified in `package.json`."
       );
     }
   }
@@ -94,8 +95,8 @@ async function readDependencies(
   // Below is for backward compatibility for the deprecated command line options
   let dependenciesFromDeprecatedArg: string[] = [];
   if (packagesFallback != null) {
-    console.warn(
-      "The `packages` argument is deprecated and will be removed in the future. Please specify `stlite.desktop.dependencies` in the package.json for that purpose."
+    deprecationWarning(
+      "The `packages` argument is deprecated and will be removed in a future release. Use `stlite.desktop.dependencies` in your `package.json` instead."
     );
     dependenciesFromDeprecatedArg = packagesFallback;
   }
@@ -129,8 +130,8 @@ async function readRequirementsTxtFilePaths(
     requirementsTxtFilePathsFallback != null &&
     requirementsTxtFilePathsFallback.length > 0
   ) {
-    console.warn(
-      "The `requirement` argument is deprecated and will be removed in the future. Please specify `stlite.desktop.requirementsTxtFiles` in the package.json for that purpose."
+    deprecationWarning(
+      "The `requirement` argument is deprecated and will be removed in a future release. Use `stlite.desktop.requirementsTxtFiles` in your `package.json` instead."
     );
     // We don't need to resolve these paths because they are given as command line arguments which are assumed to be relative to the current working directory.
     requirementsTxtFilePathsFromDeprecatedArg =

diff --git a/packages/desktop/bin-src/dump_artifacts/index.ts b/packages/desktop/bin-src/dump_artifacts/index.ts
@@ -13,6 +13,7 @@ import { dumpManifest } from "./manifest";
 import { readConfig } from "./config";
 import { validateRequirements, parseRequirementsTxt } from "@stlite/common";
 import { glob } from "glob";
+import { logger } from "./logger";
 
 // @ts-ignore
 global.fetch = fetch; // The global `fetch()` is necessary for micropip.install() to load the remote packages.
@@ -22,6 +23,9 @@ const pathFromScriptToBuild =
 const pathFromScriptToWheels =
   process.env.PATH_FROM_SCRIPT_TO_WHEELS ?? "../../wheels";
 
+/**
+ * Ensure that the given package is loaded by throwing an error if any error occurs in the loadPackage() function.
+ */
 async function ensureLoadPackage(
   pyodide: PyodideInterface,
   packageName: string | string[]
@@ -41,7 +45,7 @@ interface CopyBuildDirectoryOptions {
   copyTo: string;
 }
 async function copyBuildDirectory(options: CopyBuildDirectoryOptions) {
-  console.info(
+  logger.info(
     "Copy the build directory (the bare built app files) to this directory..."
   );
 
@@ -52,7 +56,7 @@ async function copyBuildDirectory(options: CopyBuildDirectoryOptions) {
   }
 
   if (sourceDir === options.copyTo) {
-    console.warn(
+    logger.warn(
       `sourceDir == destDir (${sourceDir}). Are you in the development environment? Skip copying the directory.`
     );
     return;
@@ -61,9 +65,7 @@ async function copyBuildDirectory(options: CopyBuildDirectoryOptions) {
   if (options.keepOld) {
     try {
       await fsPromises.access(options.copyTo);
-      console.info(
-        `${options.copyTo} already exists. Use it and skip copying.`
-      );
+      logger.info(`${options.copyTo} already exists. Use it and skip copying.`);
       return;
     } catch {
       // If the destination directory does not exist
@@ -73,7 +75,7 @@ async function copyBuildDirectory(options: CopyBuildDirectoryOptions) {
     }
   }
 
-  console.log(`Copy ${sourceDir} to ${options.copyTo}`);
+  logger.debug(`Copy ${sourceDir} to ${options.copyTo}`);
   await fsPromises.rm(options.copyTo, { recursive: true, force: true });
   await fsExtra.copy(sourceDir, options.copyTo, { errorOnExist: true });
 }
@@ -108,14 +110,14 @@ async function prepareLocalWheel(
   pyodide: PyodideInterface,
   localPath: string
 ): Promise<string> {
-  console.log(`Preparing the local wheel ${localPath}`);
+  logger.debug(`Preparing the local wheel: %s`, localPath);
 
   const data = await fsPromises.readFile(localPath);
   const emfsPath = "/tmp/" + path.basename(localPath);
   pyodide.FS.writeFile(emfsPath, data);
 
   const requirement = `emfs:${emfsPath}`;
-  console.log(`The local wheel ${localPath} is prepared as ${requirement}`);
+  logger.debug(`The local wheel %s is prepared as %s`, localPath, requirement);
   return requirement;
 }
 
@@ -143,7 +145,7 @@ async function installPackages(
   );
   requirements.push(streamlitWheel);
 
-  console.log("Install the packages:", requirements);
+  logger.info("Install the packages: %j", requirements);
   await micropip.install.callKwargs(requirements, { keep_going: true });
 }
 
@@ -155,7 +157,7 @@ interface CreateSitePackagesSnapshotOptions {
 async function createSitePackagesSnapshot(
   options: CreateSitePackagesSnapshotOptions
 ) {
-  console.info("Create the site-packages snapshot file...");
+  logger.info("Create the site-packages snapshot file...");
 
   const pyodide = await loadPyodide();
 
@@ -166,7 +168,7 @@ async function createSitePackagesSnapshot(
 
   const mockedPackages: string[] = [];
   if (options.usedPrebuiltPackages.length > 0) {
-    console.log(
+    logger.info(
       "Mocking prebuilt packages so that they will not be included in the site-packages snapshot because these will be installed from the vendored wheel files at runtime..."
     );
     options.usedPrebuiltPackages.forEach((pkg) => {
@@ -175,24 +177,22 @@ async function createSitePackagesSnapshot(
         throw new Error(`Package ${pkg} is not found in the lock file.`);
       }
 
-      console.log(`Mock ${packageInfo.name} ${packageInfo.version}`);
+      logger.debug(`Mock ${packageInfo.name} ${packageInfo.version}`);
       micropip.add_mock_package(packageInfo.name, packageInfo.version);
       mockedPackages.push(packageInfo.name);
     });
   }
 
-  console.log(
-    `Install the requirements ${JSON.stringify(options.requirements)}`
-  );
+  logger.info(`Install the requirements %j`, options.requirements);
 
   await installPackages(pyodide, {
     requirements: options.requirements,
   });
 
-  console.log("Remove the mocked packages", mockedPackages);
+  logger.info("Remove the mocked packages: %j", mockedPackages);
   mockedPackages.forEach((pkg) => micropip.remove_mock_package(pkg));
 
-  console.log("Archive the site-packages director(y|ies)");
+  logger.info("Archive the site-packages director(y|ies)");
   const archiveFilePath = "/tmp/site-packages-snapshot.tar.gz";
   await pyodide.runPythonAsync(`
     import os
@@ -209,10 +209,10 @@ async function createSitePackagesSnapshot(
             gzf.add(site_packages)
   `);
 
-  console.log("Extract the archive file from EMFS");
+  logger.info("Extract the archive file from EMFS");
   const archiveBin = pyodide.FS.readFile(archiveFilePath);
 
-  console.log(`Save the archive file (${options.saveTo})`);
+  logger.info(`Save the archive file (${options.saveTo})`);
   await fsPromises.writeFile(options.saveTo, archiveBin);
 }
 
@@ -223,7 +223,7 @@ interface CopyAppDirectoryOptions {
 }
 
 async function copyAppDirectory(options: CopyAppDirectoryOptions) {
-  console.info("Copy the app directory...");
+  logger.info("Copy the app directory...");
 
   await Promise.all(
     options.filePathPatterns.map(async (pattern) => {
@@ -233,7 +233,7 @@ async function copyAppDirectory(options: CopyAppDirectoryOptions) {
       });
 
       if (fileRelPaths.length === 0) {
-        console.warn(
+        logger.warn(
           `No files match the pattern "${pattern}" in "${options.cwd}".`
         );
         return;
@@ -243,7 +243,7 @@ async function copyAppDirectory(options: CopyAppDirectoryOptions) {
         fileRelPaths.map(async (relPath) => {
           const srcPath = path.resolve(options.cwd, relPath);
           const destPath = path.resolve(options.buildAppDirectory, relPath);
-          console.log(`Copy ${srcPath} to ${destPath}`);
+          logger.debug(`Copy ${srcPath} to ${destPath}`);
           await fsExtra.copy(srcPath, destPath, {
             errorOnExist: true,
           });
@@ -300,15 +300,15 @@ async function downloadPrebuiltPackageWheels(
     makePyodideUrl(pkg.file_name)
   );
 
-  console.log("Downloading the used prebuilt packages...");
+  logger.info("Downloading the used prebuilt packages...");
   await Promise.all(
     usedPrebuiltPackageUrls.map(async (pkgUrl) => {
       const dstPath = path.resolve(
         options.destDir,
         "./pyodide",
         path.basename(pkgUrl)
       );
-      console.log(`Download ${pkgUrl} to ${dstPath}`);
+      logger.debug(`Download ${pkgUrl} to ${dstPath}`);
       const res = await fetch(pkgUrl);
       if (!res.ok) {
         throw new Error(
@@ -325,10 +325,7 @@ yargs(hideBin(process.argv))
   .command(
     "* [appHomeDirSource] [packages..]",
     "Put the user code and data and the snapshot of the required packages into the build artifact.",
-    () => {},
-    (argv) => {
-      console.info(argv);
-    }
+    () => {}
   )
   .positional("appHomeDirSource", {
     describe:
@@ -362,8 +359,18 @@ yargs(hideBin(process.argv))
     alias: "k",
     describe: "Keep the existing build directory contents except appHomeDir.",
   })
+  .options("logLevel", {
+    type: "string",
+    default: "info",
+    describe: "The log level",
+    choices: ["debug", "info", "warn", "error"],
+  })
   .parseAsync()
   .then(async (args) => {
+    logger.level = args.logLevel;
+
+    logger.debug(`Command line arguments: %j`, args);
+
     const projectDir = args.project;
     const destDir = path.resolve(projectDir, "./build");
 
@@ -379,32 +386,37 @@ yargs(hideBin(process.argv))
         requirementsTxtFilePaths: args.requirement,
       },
     });
-    console.log("File/directory patterns to be included:", config.files);
-    console.log("Entrypoint:", config.entrypoint);
-    console.log("Dependencies:", config.dependencies);
-    console.log("`requirements.txt` files:", config.requirementsTxtFilePaths);
+    logger.info(`File/directory patterns to be included: %j`, config.files);
+    logger.info(`Entrypoint: %s`, config.entrypoint);
+    logger.info(`Dependencies: %j`, config.dependencies);
+    logger.info(
+      `\`requirements.txt\` files: %j`,
+      config.requirementsTxtFilePaths
+    );
 
     const dependenciesFromRequirementsTxt = await Promise.all(
       config.requirementsTxtFilePaths.map(async (requirementsTxtPath) => {
         return readRequirements(requirementsTxtPath);
       })
     ).then((parsedRequirements) => parsedRequirements.flat());
-    console.log(
-      "Dependencies from `requirements.txt` files:",
+    logger.info(
+      "Dependencies from `requirements.txt` files: %j",
       dependenciesFromRequirementsTxt
     );
 
     const dependencies = validateRequirements([
       ...config.dependencies,
       ...dependenciesFromRequirementsTxt,
     ]);
-    console.log("Validated dependency list:", dependencies);
+    logger.info("Validated dependency list: %j", dependencies);
 
     const usedPrebuiltPackages = await inspectUsedPrebuiltPackages({
       requirements: dependencies,
     });
-    console.log("The prebuilt packages loaded for the given requirements:");
-    console.log(usedPrebuiltPackages);
+    logger.info(
+      "The prebuilt packages loaded for the given requirements: %j",
+      usedPrebuiltPackages
+    );
 
     await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild });
 

diff --git a/packages/desktop/bin-src/dump_artifacts/logger.ts b/packages/desktop/bin-src/dump_artifacts/logger.ts
@@ -0,0 +1,18 @@
+import * as winston from "winston";
+
+export const logger = winston.createLogger({
+  transports: [
+    new winston.transports.Console({
+      format: winston.format.combine(
+        winston.format.splat(),
+        winston.format.cli()
+      ),
+    }),
+  ],
+});
+
+export function deprecationWarning(message: string) {
+  process.emitWarning(message, {
+    type: "DeprecationWarning",
+  });
+}
diff --git a/packages/desktop/bin-src/dump_artifacts/manifest.ts b/packages/desktop/bin-src/dump_artifacts/manifest.ts
@@ -4,6 +4,7 @@ import {
   type DesktopAppManifest,
   DesktopAppManifestStruct,
 } from "../../electron/manifest";
+import { logger } from "./logger";
 
 export function coerceDesktopAppManifest(obj: unknown): DesktopAppManifest {
   const manifestData = s.mask(obj ?? {}, DesktopAppManifestStruct);
@@ -38,10 +39,10 @@ export async function dumpManifest(options: DumpManifestOptions) {
     ...options.packageJsonStliteDesktopField,
     ...options.fallbacks,
   });
+  logger.info(`App manifest: %j`, manifestData);
 
   const manifestDataStr = JSON.stringify(manifestData, null, 2);
-  console.log(`Dump the manifest file -> ${options.manifestFilePath}`);
-  console.log(manifestDataStr);
+  logger.info(`Dump the manifest file -> ${options.manifestFilePath}`);
   await fsPromises.writeFile(options.manifestFilePath, manifestDataStr, {
     encoding: "utf-8",
   });

diff --git a/packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts b/packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts
@@ -1,5 +1,6 @@
 import fetch from "node-fetch";
 import { makePyodideUrl } from "./url";
+import { logger } from "./logger";
 
 interface PackageInfo {
   name: string;
@@ -18,7 +19,7 @@ export class PrebuiltPackagesData {
   > {
     const url = makePyodideUrl("pyodide-lock.json");
 
-    console.log(`Load the Pyodide pyodide-lock.json from ${url}`);
+    logger.info(`Load the Pyodide pyodide-lock.json from ${url}`);
     const res = await fetch(url, undefined);
     const resJson = await res.json();
 

diff --git a/packages/desktop/package.json b/packages/desktop/package.json
@@ -93,7 +93,8 @@
     "react-toastify": "^9.1.1",
     "superstruct": "^1.0.4",
     "ts-node": "^10.9.1",
-    "typescript": "^4.9.4"
+    "typescript": "^4.9.4",
+    "winston": "^3.13.0"
   },
   "///": "build.productName is necessary because electron-builder uses the package name for its purpose but the scoped name including '@' makes a problem: https://github.com/electron-userland/electron-builder/issues/3230",
   "build": {