Add file:// schema to the Pyodide ESM path in the desktop package (#959)

* Add file:// schema to the Pyodide ESM path in the desktop package

* Add comment

* Fix to attach the file:// scheme only to the url passed to import()

* Add .ports to MessageEventLike for NodeJS worker mode

* Fix

* Fix

* Write tests for pyodide load url resolution

* Add comment

packages/desktop/electron/main.ts

@@ -121,14 +121,14 @@ const createWindow = async () => {
     }
 
     // Use the ESM version of Pyodide because `importScripts()` can't be used in this environment.
-    const defaultPyodideUrl = path.resolve(__dirname, "../pyodide/pyodide.mjs");
+    const pyodidePath = path.resolve(__dirname, "..", "pyodide", "pyodide.mjs"); // For Windows compatibility, rely on path.resolve() to join the path elements.
 
     function onMessageFromWorker(value: any) {
       mainWindow.webContents.send("messageFromNodeJsWorker", value);
     }
-    worker = new workerThreads.Worker(path.resolve(__dirname, "./worker.js"), {
+    worker = new workerThreads.Worker(path.resolve(__dirname, "worker.js"), {
       env: {
-        PYODIDE_URL: defaultPyodideUrl,
+        PYODIDE_URL: pyodidePath,
         ...(manifest.nodefsMountpoints && {
           NODEFS_MOUNTPOINTS: JSON.stringify(manifest.nodefsMountpoints),
         }),

packages/desktop/src/nodejs-worker.ts

@@ -2,6 +2,7 @@ export const USE_NODEJS_WORKER = window.nodeJsWorkerAPI.USE_NODEJS_WORKER;
 
 interface MessageEventLike<T = any> {
   readonly data: T;
+  readonly ports: MessagePort[];
 }
 
 export class NodeJsWorkerMock {
@@ -11,7 +12,7 @@ export class NodeJsWorkerMock {
     this.initializePromise = window.nodeJsWorkerAPI.initialize();
 
     window.nodeJsWorkerAPI.onMessage((data: unknown) => {
-      this.onmessage && this.onmessage({ data });
+      this.onmessage && this.onmessage({ data, ports: [] });
     });
   }
 

packages/kernel/src/pyodide-loader.test.ts

@@ -0,0 +1,44 @@
+import { describe, it, expect } from "vitest";
+import { resolvePyodideUrl } from "./pyodide-loader";
+
+describe("resolvePyodideUrl", () => {
+  it("resolves a non-ESM remote URL", async () => {
+    const result = await resolvePyodideUrl(
+      "https://cdn.jsdelivr.net/pyodide/v0.26.0/full/pyodide.js",
+    );
+    expect(result).toEqual({
+      scriptURL: "https://cdn.jsdelivr.net/pyodide/v0.26.0/full/pyodide.js",
+      pyodideIndexURL: "https://cdn.jsdelivr.net/pyodide/v0.26.0/full/",
+      isESModule: false,
+    });
+  });
+
+  it("resolves an ESM remote URL", async () => {
+    const result = await resolvePyodideUrl(
+      "https://cdn.jsdelivr.net/pyodide/v0.26.0/full/pyodide.mjs",
+    );
+    expect(result).toEqual({
+      scriptURL: "https://cdn.jsdelivr.net/pyodide/v0.26.0/full/pyodide.mjs",
+      pyodideIndexURL: "https://cdn.jsdelivr.net/pyodide/v0.26.0/full/",
+      isESModule: true,
+    });
+  });
+
+  it("resolves a non-ESM local URL", async () => {
+    const result = await resolvePyodideUrl("/path/to/pyodide.js");
+    expect(result).toEqual({
+      scriptURL: "/path/to/pyodide.js",
+      pyodideIndexURL: "/path/to/",
+      isESModule: false,
+    });
+  });
+
+  it("resolves an ESM local URL", async () => {
+    const result = await resolvePyodideUrl("/path/to/pyodide.mjs");
+    expect(result).toEqual({
+      scriptURL: "file:///path/to/pyodide.mjs", // import() requires the `file://` scheme on Windows
+      pyodideIndexURL: "/path/to/",
+      isESModule: true,
+    });
+  });
+});

packages/kernel/src/pyodide-loader.ts

@@ -0,0 +1,69 @@
+import type Pyodide from "pyodide";
+
+interface ResolvePyodideUrlResult {
+  scriptURL: string;
+  pyodideIndexURL: string;
+  isESModule: boolean;
+}
+export async function resolvePyodideUrl(
+  pyodideUrl: string,
+): Promise<ResolvePyodideUrlResult> {
+  const isNode = typeof process !== "undefined" && process.versions?.node;
+
+  let sep: string;
+  if (isNode) {
+    const nodePath = await import(/* webpackIgnore: true */ "node:path");
+    sep = nodePath.sep;
+  } else {
+    sep = "/"; // URL path separator
+  }
+
+  // Ref: https://github.com/jupyterlite/pyodide-kernel/blob/v0.1.3/packages/pyodide-kernel/src/kernel.ts#L55
+  const pyodideIndexURL = pyodideUrl.slice(0, pyodideUrl.lastIndexOf(sep) + 1);
+
+  // Ref: https://github.com/jupyterlite/pyodide-kernel/blob/v0.1.3/packages/pyodide-kernel/src/worker.ts#L40-L54
+  if (pyodideUrl.endsWith(".mjs")) {
+    if (isNode) {
+      // Special care for Node.js on Windows because the `file://` scheme is required in the URL passed to import() on Windows. See https://github.com/whitphx/stlite/issues/957
+      const nodePath = await import(/* webpackIgnore: true */ "node:path");
+      const nodeUrl = await import(/* webpackIgnore: true */ "node:url");
+      const possiblyLocalFilePath = !pyodideUrl.includes("://");
+      if (possiblyLocalFilePath && nodePath.isAbsolute(pyodideUrl)) {
+        pyodideUrl = nodeUrl.pathToFileURL(pyodideUrl).href;
+      }
+    }
+    return {
+      scriptURL: pyodideUrl,
+      pyodideIndexURL,
+      isESModule: true,
+    };
+  } else {
+    return {
+      scriptURL: pyodideUrl,
+      pyodideIndexURL,
+      isESModule: false,
+    };
+  }
+}
+
+export async function initPyodide(
+  pyodideUrl: string,
+  loadPyodideOptions: Parameters<typeof Pyodide.loadPyodide>[0],
+): Promise<Pyodide.PyodideInterface> {
+  const { scriptURL, pyodideIndexURL, isESModule } =
+    await resolvePyodideUrl(pyodideUrl);
+
+  // Ref: https://github.com/jupyterlite/pyodide-kernel/blob/v0.1.3/packages/pyodide-kernel/src/worker.ts#L40-L54
+  let loadPyodide: typeof Pyodide.loadPyodide;
+  if (isESModule) {
+    // note: this does not work at all in firefox
+    const pyodideModule: typeof Pyodide = await import(
+      /* webpackIgnore: true */ scriptURL
+    );
+    loadPyodide = pyodideModule.loadPyodide;
+  } else {
+    importScripts(scriptURL);
+    loadPyodide = (self as any).loadPyodide;
+  }
+  return loadPyodide({ ...loadPyodideOptions, indexURL: pyodideIndexURL });
+}

packages/kernel/src/worker-runtime.ts

@@ -3,6 +3,7 @@ import type { PyProxy, PyBuffer } from "pyodide/ffi";
 import { PromiseDelegate } from "@stlite/common";
 import { writeFileWithParents, renameWithParents } from "./file";
 import { validateRequirements } from "@stlite/common/src/requirements";
+import { initPyodide } from "./pyodide-loader";
 import { mockPyArrow } from "./mock";
 import { tryModuleAutoLoad } from "./module-auto-load";
 import type {
@@ -13,28 +14,6 @@ import type {
   PyodideConvertiblePrimitive,
 } from "./types";
 
-async function initPyodide(
-  pyodideUrl: string,
-  loadPyodideOptions: Parameters<typeof Pyodide.loadPyodide>[0],
-): Promise<Pyodide.PyodideInterface> {
-  // Ref: https://github.com/jupyterlite/pyodide-kernel/blob/v0.1.3/packages/pyodide-kernel/src/kernel.ts#L55
-  const indexUrl = pyodideUrl.slice(0, pyodideUrl.lastIndexOf("/") + 1);
-
-  // Ref: https://github.com/jupyterlite/pyodide-kernel/blob/v0.1.3/packages/pyodide-kernel/src/worker.ts#L40-L54
-  let loadPyodide: typeof Pyodide.loadPyodide;
-  if (pyodideUrl.endsWith(".mjs")) {
-    // note: this does not work at all in firefox
-    const pyodideModule: typeof Pyodide = await import(
-      /* webpackIgnore: true */ pyodideUrl
-    );
-    loadPyodide = pyodideModule.loadPyodide;
-  } else {
-    importScripts(pyodideUrl);
-    loadPyodide = (self as any).loadPyodide;
-  }
-  return loadPyodide({ ...loadPyodideOptions, indexURL: indexUrl });
-}
-
 export type PostMessageFn = (message: OutMessage, port?: MessagePort) => void;
 
 const self = global as typeof globalThis & {