From 0c6760d22724fe56564a204f0324dd345a5ef891 Mon Sep 17 00:00:00 2001
From: "Yuichiro Tachibana (Tsuchiya)" <t.yic.yt@gmail.com>
Date: Wed, 5 Jun 2024 16:07:20 +0100
Subject: [PATCH] Auto-install the imported packages (#902)
* Add autoInstall option to the kernel
* Run pyodide.loadPackagesFromImports() at init and file writing
* Send the auto-installed packages to the editor and update the requirements tab content
* Update the auto install event emits a promise to resolve the installed package list
* Update to call loadPackage() only when needed
* Improve the toast notification
* Fix
* Fix
* Update the editor content
* Fix
* Implement module auto-load at each run and split the autoInstall option into moduleAutoLoadOnRun and moduleAutoLoadOnSave
* Revert "Implement module auto-load at each run and split the autoInstall option into moduleAutoLoadOnRun and moduleAutoLoadOnSave"
This reverts commit 61cfd1c72c58be4a0db8d4aaa7b5197696b05d89.
* Rename auto-install to module-auto-load
* Update findImports impl
* Fix to await the auto-load promise in the script runner
* Fix
* Rename messages
* Fix
* Apply formatter
* Add comment
* Refactoring
* Refactoring
* Fix inter-window messaging
* Fix
* Fix
* Fix <Editor /> to handle addRequirements() in an imperative way
---
.../stlite-kernel-with-toast.tsx | 74 ++-
packages/desktop/electron/worker.ts | 9 +-
packages/kernel/src/kernel.ts | 46 +-
packages/kernel/src/module-auto-load.ts | 71 +++
packages/kernel/src/types.ts | 27 +-
packages/kernel/src/worker-runtime.ts | 50 +-
packages/kernel/src/worker.ts | 3 +-
packages/sharing-common/src/messages.ts | 24 +-
packages/sharing-editor/src/App.tsx | 44 +-
.../src/Editor/components/Tab.module.scss | 2 +-
packages/sharing-editor/src/Editor/index.tsx | 456 ++++++++++--------
.../src/StliteSharingIFrame/index.tsx | 24 +-
packages/sharing/src/App.tsx | 31 +-
streamlit | 2 +-
14 files changed, 607 insertions(+), 256 deletions(-)
create mode 100644 packages/kernel/src/module-auto-load.ts
diff --git a/packages/common-react/src/toastify-components/stlite-kernel-with-toast.tsx b/packages/common-react/src/toastify-components/stlite-kernel-with-toast.tsx
index 7293fdf29..0892c3f7a 100644
--- a/packages/common-react/src/toastify-components/stlite-kernel-with-toast.tsx
+++ b/packages/common-react/src/toastify-components/stlite-kernel-with-toast.tsx
@@ -1,34 +1,36 @@
import React from "react";
-import { StliteKernel } from "@stlite/kernel";
-import { toast } from "react-toastify";
+import type { StliteKernel, StliteKernelOptions } from "@stlite/kernel";
+import { toast, ToastPromiseParams } from "react-toastify";
import ErrorToastContent from "./ErrorToastContent";
-type ToastPromiseParams<TData> = Parameters<typeof toast.promise<TData>>;
-type ToastPromiseMessages<TData> = Partial<
- Record<keyof ToastPromiseParams<TData>[1], string>
->;
-type ToastPromiseReturnType = ReturnType<typeof toast.promise>;
-function stliteStyledPromiseToast<TData>(
- promise: ToastPromiseParams<TData>[0],
- messages: ToastPromiseMessages<TData>
-): ToastPromiseReturnType {
+function stliteStyledPromiseToast<
+ TData = unknown,
+ TError extends Error | undefined = undefined,
+ TPending = unknown
+>(
+ promise: Promise<TData>,
+ messages: ToastPromiseParams<TData, TError, TPending>
+): ReturnType<typeof toast.promise> {
const errorMessage = messages.error;
- return toast.promise<TData, Error>(
+ return toast.promise<TData, TError, TPending>(
promise,
{
pending: messages.pending,
success: messages.success,
- error: errorMessage && {
- render({ data }) {
- return data ? (
- <ErrorToastContent message={errorMessage} error={data} />
- ) : (
- messages.error
- );
- },
- autoClose: false,
- closeOnClick: false,
- },
+ error:
+ typeof errorMessage === "string"
+ ? {
+ render({ data }) {
+ return data ? (
+ <ErrorToastContent message={errorMessage} error={data} />
+ ) : (
+ <>messages.error</>
+ );
+ },
+ autoClose: false,
+ closeOnClick: false,
+ }
+ : errorMessage,
},
{
hideProgressBar: true,
@@ -37,8 +39,32 @@ function stliteStyledPromiseToast<TData>(
);
}
+export interface StliteKernelWithToastOptions {
+ onModuleAutoLoad?: StliteKernelOptions["onModuleAutoLoad"];
+}
export class StliteKernelWithToast {
- constructor(private kernel: StliteKernel) {}
+ constructor(
+ private kernel: StliteKernel,
+ options?: StliteKernelWithToastOptions
+ ) {
+ kernel.onModuleAutoLoad = (packagesToLoad, installPromise) => {
+ if (options?.onModuleAutoLoad) {
+ options.onModuleAutoLoad(packagesToLoad, installPromise);
+ }
+
+ stliteStyledPromiseToast(installPromise, {
+ success: {
+ render({ data }) {
+ return `Auto-loaded${
+ data ? ": " + data.map((pkg) => pkg.name).join(", ") : " packages"
+ }`;
+ },
+ },
+ error: "Failed to auto-load packages",
+ pending: "Auto-loading packages",
+ });
+ };
+ }
public writeFile(...args: Parameters<StliteKernel["writeFile"]>) {
return stliteStyledPromiseToast<void>(this.kernel.writeFile(...args), {
diff --git a/packages/desktop/electron/worker.ts b/packages/desktop/electron/worker.ts
index 6ab704cfc..1b9e12482 100644
--- a/packages/desktop/electron/worker.ts
+++ b/packages/desktop/electron/worker.ts
@@ -1,11 +1,14 @@
import { parentPort } from "node:worker_threads";
-import { startWorkerEnv } from "@stlite/kernel/src/worker-runtime";
+import {
+ startWorkerEnv,
+ type PostMessageFn,
+} from "@stlite/kernel/src/worker-runtime";
import { loadNodefsMountpoints } from "./worker-options";
-function postMessage(value: any) {
+const postMessage: PostMessageFn = (value) => {
console.debug("[worker thread] postMessage from worker", value);
parentPort?.postMessage(value);
-}
+};
const handleMessage = startWorkerEnv(
process.env.PYODIDE_URL as string,
diff --git a/packages/kernel/src/kernel.ts b/packages/kernel/src/kernel.ts
index 62d230408..0d840cea8 100644
--- a/packages/kernel/src/kernel.ts
+++ b/packages/kernel/src/kernel.ts
@@ -1,5 +1,6 @@
// Ref: https://github.com/jupyterlite/jupyterlite/blob/f2ecc9cf7189cb19722bec2f0fc7ff5dfd233d47/packages/pyolite-kernel/src/kernel.ts
+import type { PackageData } from "pyodide";
import { PromiseDelegate } from "@stlite/common";
import type { IHostConfigResponse } from "@streamlit/lib/src/hostComm/types";
@@ -21,6 +22,7 @@ import type {
StliteWorker,
WorkerInitialData,
StreamlitConfig,
+ ModuleAutoLoadMessage,
} from "./types";
import { assertStreamlitConfig } from "./types";
@@ -117,6 +119,13 @@ export interface StliteKernelOptions {
idbfsMountpoints?: WorkerInitialData["idbfsMountpoints"];
+ moduleAutoLoad?: WorkerInitialData["moduleAutoLoad"];
+
+ onModuleAutoLoad?: (
+ packagesToLoad: string[],
+ installPromise: Promise<PackageData[]>
+ ) => void;
+
onProgress?: (message: string) => void;
onLoad?: () => void;
@@ -143,11 +152,10 @@ export class StliteKernel {
public readonly hostConfigResponse: IHostConfigResponse; // Will be passed to ConnectionManager to call `onHostConfigResp` from it.
- private onProgress: StliteKernelOptions["onProgress"];
-
- private onLoad: StliteKernelOptions["onLoad"];
-
- private onError: StliteKernelOptions["onError"];
+ public onProgress: StliteKernelOptions["onProgress"];
+ public onLoad: StliteKernelOptions["onLoad"];
+ public onError: StliteKernelOptions["onError"];
+ public onModuleAutoLoad: StliteKernelOptions["onModuleAutoLoad"];
constructor(options: StliteKernelOptions) {
this.basePath = (options.basePath ?? window.location.pathname)
@@ -157,6 +165,7 @@ export class StliteKernel {
this.onProgress = options.onProgress;
this.onLoad = options.onLoad;
this.onError = options.onError;
+ this.onModuleAutoLoad = options.onModuleAutoLoad;
if (options.worker) {
this._worker = options.worker;
@@ -168,7 +177,8 @@ export class StliteKernel {
}
this._worker.onmessage = (e) => {
- this._processWorkerMessage(e.data);
+ const messagePort: MessagePort | undefined = e.ports[0];
+ this._processWorkerMessage(e.data, messagePort);
};
let wheels: WorkerInitialData["wheels"] = undefined;
@@ -209,6 +219,7 @@ export class StliteKernel {
options.mountedSitePackagesSnapshotFilePath,
streamlitConfig: options.streamlitConfig,
idbfsMountpoints: options.idbfsMountpoints,
+ moduleAutoLoad: options.moduleAutoLoad ?? false,
};
}
@@ -337,7 +348,7 @@ export class StliteKernel {
*
* @param msg The worker message to process.
*/
- private _processWorkerMessage(msg: OutMessage): void {
+ private _processWorkerMessage(msg: OutMessage, port?: MessagePort): void {
switch (msg.type) {
case "event:start": {
this._worker.postMessage({
@@ -364,6 +375,27 @@ export class StliteKernel {
this.handleWebSocketMessage && this.handleWebSocketMessage(payload);
break;
}
+ case "event:moduleAutoLoad": {
+ if (port == null) {
+ throw new Error("Port is required for moduleAutoLoad event");
+ }
+ this.onModuleAutoLoad &&
+ this.onModuleAutoLoad(
+ msg.data.packagesToLoad,
+ new Promise((resolve, reject) => {
+ port.onmessage = (e) => {
+ const msg: ModuleAutoLoadMessage = e.data;
+ if (msg.type === "moduleAutoLoad:success") {
+ resolve(msg.data.loadedPackages);
+ } else {
+ reject(msg.error);
+ }
+ port.close();
+ };
+ })
+ );
+ break;
+ }
}
}
diff --git a/packages/kernel/src/module-auto-load.ts b/packages/kernel/src/module-auto-load.ts
new file mode 100644
index 000000000..69f1f04be
--- /dev/null
+++ b/packages/kernel/src/module-auto-load.ts
@@ -0,0 +1,71 @@
+import type { PackageData, PyodideInterface } from "pyodide";
+import type { ModuleAutoLoadMessage } from "./types";
+import type { PostMessageFn } from "./worker-runtime";
+
+function findImports(pyodide: PyodideInterface, source: string): string[] {
+ return pyodide.pyodide_py.ffi._pyodide._base
+ .find_imports(source)
+ .toJs() as string[];
+}
+
+export async function tryModuleAutoLoad(
+ pyodide: PyodideInterface,
+ postMessage: PostMessageFn,
+ sources: string[]
+): Promise<void> {
+ // Ref: `pyodide.loadPackagesFromImports` (https://github.com/pyodide/pyodide/blob/0.26.0/src/js/api.ts#L191)
+
+ const importsArr = sources.map((source) => findImports(pyodide, source));
+ const imports = Array.from(new Set(importsArr.flat()));
+
+ const notFoundImports = imports.filter(
+ (name) =>
+ !pyodide.runPython(`__import__('importlib').util.find_spec('${name}')`)
+ );
+
+ const packagesToLoad = notFoundImports
+ .map((name) =>
+ (
+ pyodide as unknown as {
+ _api: { _import_name_to_package_name: Map<string, string> };
+ }
+ )._api._import_name_to_package_name.get(name)
+ )
+ .filter((name) => name) as string[];
+
+ if (packagesToLoad.length === 0) {
+ return;
+ }
+
+ const channel = new MessageChannel();
+
+ postMessage(
+ {
+ type: "event:moduleAutoLoad",
+ data: {
+ packagesToLoad,
+ },
+ },
+ channel.port2
+ );
+
+ try {
+ const loadedPackages = await pyodide.loadPackage(packagesToLoad);
+
+ channel.port1.postMessage({
+ type: "moduleAutoLoad:success",
+ data: {
+ loadedPackages,
+ },
+ } as ModuleAutoLoadMessage);
+ channel.port1.close();
+ return;
+ } catch (error) {
+ channel.port1.postMessage({
+ type: "moduleAutoLoad:error",
+ error: error as Error,
+ } as ModuleAutoLoadMessage);
+ channel.port1.close();
+ throw error;
+ }
+}
diff --git a/packages/kernel/src/types.ts b/packages/kernel/src/types.ts
index 35179673c..31434bfbb 100644
--- a/packages/kernel/src/types.ts
+++ b/packages/kernel/src/types.ts
@@ -1,4 +1,4 @@
-import type { PyodideInterface } from "pyodide";
+import type { PyodideInterface, PackageData } from "pyodide";
export type PyodideConvertiblePrimitive =
| string
@@ -57,6 +57,7 @@ export interface WorkerInitialData {
streamlitConfig?: StreamlitConfig;
idbfsMountpoints?: string[];
nodefsMountpoints?: Record<string, string>;
+ moduleAutoLoad: boolean;
}
/**
@@ -161,12 +162,34 @@ export interface OutMessageWebSocketBack extends OutMessageBase {
payload: Uint8Array | string;
};
}
+export interface OutMessageModuleAutoLoadEvent extends OutMessageBase {
+ type: "event:moduleAutoLoad";
+ data: {
+ packagesToLoad: string[];
+ };
+}
export type OutMessage =
| OutMessageStartEvent
| OutMessageProgressEvent
| OutMessageErrorEvent
| OutMessageLoadedEvent
- | OutMessageWebSocketBack;
+ | OutMessageWebSocketBack
+ | OutMessageModuleAutoLoadEvent;
+
+export interface ModuleAutoLoadMessageBase {
+ type: string;
+}
+export interface ModuleAutoLoadSuccess extends ModuleAutoLoadMessageBase {
+ type: "moduleAutoLoad:success";
+ data: {
+ loadedPackages: PackageData[];
+ };
+}
+export interface ModuleAutoLoadError extends ModuleAutoLoadMessageBase {
+ type: "moduleAutoLoad:error";
+ error: Error;
+}
+export type ModuleAutoLoadMessage = ModuleAutoLoadSuccess | ModuleAutoLoadError;
/**
* Reply message to InMessage
diff --git a/packages/kernel/src/worker-runtime.ts b/packages/kernel/src/worker-runtime.ts
index 3cc8875d4..f2b9dc3f0 100644
--- a/packages/kernel/src/worker-runtime.ts
+++ b/packages/kernel/src/worker-runtime.ts
@@ -4,6 +4,7 @@ import { PromiseDelegate } from "@stlite/common";
import { writeFileWithParents, renameWithParents } from "./file";
import { validateRequirements } from "@stlite/common/src/requirements";
import { mockPyArrow } from "./mock";
+import { tryModuleAutoLoad } from "./module-auto-load";
import type {
WorkerInitialData,
OutMessage,
@@ -34,15 +35,34 @@ async function initPyodide(
return loadPyodide({ ...loadPyodideOptions, indexURL: indexUrl });
}
+export type PostMessageFn = (message: OutMessage, port?: MessagePort) => void;
+
const self = global as typeof globalThis & {
__logCallback__: (levelno: number, msg: string) => void;
__streamlitFlagOptions__: Record<string, PyodideConvertiblePrimitive>;
__scriptFinishedCallback__: () => void;
+ __moduleAutoLoadPromise__: Promise<unknown> | undefined;
};
+function dispatchModuleAutoLoading(
+ pyodide: Pyodide.PyodideInterface,
+ postMessage: PostMessageFn,
+ sources: string[]
+): void {
+ const autoLoadPromise = tryModuleAutoLoad(pyodide, postMessage, sources);
+ // `autoInstallPromise` will be awaited in the script_runner on the Python side.
+ self.__moduleAutoLoadPromise__ = autoLoadPromise;
+ pyodide.runPythonAsync(`
+from streamlit.runtime.scriptrunner import script_runner
+from js import __moduleAutoLoadPromise__
+
+script_runner.moduleAutoLoadPromise = __moduleAutoLoadPromise__
+`);
+}
+
export function startWorkerEnv(
defaultPyodideUrl: string,
- postMessage: (message: OutMessage) => void,
+ postMessage: PostMessageFn,
presetInitialData?: Partial<WorkerInitialData>
) {
function postProgressMessage(message: string): void {
@@ -88,6 +108,7 @@ export function startWorkerEnv(
streamlitConfig,
idbfsMountpoints,
nodefsMountpoints,
+ moduleAutoLoad,
} = initData;
const requirements = validateRequirements(unvalidatedRequirements); // Blocks the not allowed wheel URL schemes.
@@ -133,6 +154,7 @@ export function startWorkerEnv(
// Mount files
postProgressMessage("Mounting files.");
+ const pythonFilePaths: string[] = [];
await Promise.all(
Object.keys(files).map(async (path) => {
const file = files[path];
@@ -150,6 +172,10 @@ export function startWorkerEnv(
console.debug(`Write a file "${path}"`);
writeFileWithParents(pyodide, path, data, opts);
+
+ if (path.endsWith(".py")) {
+ pythonFilePaths.push(path);
+ }
})
);
@@ -241,6 +267,12 @@ with tarfile.open("${mountedSitePackagesSnapshotFilePath}", "r") as tar_gz_file:
await micropip.install.callKwargs(requirements, { keep_going: true });
console.debug("Installed the requirements");
}
+ if (moduleAutoLoad) {
+ const sources = pythonFilePaths.map((path) =>
+ pyodide.FS.readFile(path, { encoding: "utf8" })
+ );
+ dispatchModuleAutoLoading(pyodide, postMessage, sources);
+ }
// The following code is necessary to avoid errors like `NameError: name '_imp' is not defined`
// at importing installed packages.
@@ -422,6 +454,8 @@ server.start()
postMessage({
type: "event:loaded",
});
+
+ return initData;
}
const pyodideReadyPromise = loadPyodideAndPackages().catch((error) => {
@@ -448,7 +482,7 @@ server.start()
return;
}
- await pyodideReadyPromise;
+ const { moduleAutoLoad } = await pyodideReadyPromise;
const messagePort = event.ports[0];
@@ -543,6 +577,18 @@ server.start()
case "file:write": {
const { path, data: fileData, opts } = msg.data;
+ if (
+ moduleAutoLoad &&
+ typeof fileData === "string" &&
+ path.endsWith(".py")
+ ) {
+ // Auto-install must be dispatched before writing the file
+ // because its promise should be set before saving the file triggers rerunning.
+ console.debug(`Auto install the requirements in ${path}`);
+
+ dispatchModuleAutoLoading(pyodide, postMessage, [fileData]);
+ }
+
console.debug(`Write a file "${path}"`);
writeFileWithParents(pyodide, path, fileData, opts);
messagePort.postMessage({
diff --git a/packages/kernel/src/worker.ts b/packages/kernel/src/worker.ts
index 6be09cef7..66eb07390 100644
--- a/packages/kernel/src/worker.ts
+++ b/packages/kernel/src/worker.ts
@@ -1,6 +1,7 @@
import { startWorkerEnv } from "./worker-runtime";
+const postMessage = (self as DedicatedWorkerGlobalScope).postMessage;
self.onmessage = startWorkerEnv(
"https://cdn.jsdelivr.net/pyodide/v0.26.0/full/pyodide.js",
- self.postMessage
+ (event, port) => postMessage(event, port ? [port] : undefined)
);
diff --git a/packages/sharing-common/src/messages.ts b/packages/sharing-common/src/messages.ts
index e0d47409c..c0e2057b4 100644
--- a/packages/sharing-common/src/messages.ts
+++ b/packages/sharing-common/src/messages.ts
@@ -1,3 +1,5 @@
+import type { PackageData } from "pyodide";
+
/**
* Messages from editor to app
*/
@@ -38,9 +40,27 @@ export type ForwardMessage =
| InstallMessage;
/**
- * Reply message
+ * Reply to a forward message.
*/
export interface ReplyMessage {
type: "reply";
- error?: any;
+ error?: Error;
}
+
+/**
+ * Messages from app to editor
+ */
+export interface BackwardMessageBase {
+ type: string;
+ data?: unknown;
+ stlite: true; // To distinguish from other messages such as from the Streamlit app like https://github.com/streamlit/streamlit/blob/1.35.0/frontend/lib/src/hostComm/types.ts#L49
+}
+export interface ModuleAutoLoadSuccessMessage extends BackwardMessageBase {
+ type: "moduleAutoLoadSuccess";
+ data: {
+ packagesToLoad: string[];
+ loadedPackages: PackageData[];
+ };
+}
+
+export type BackwardMessage = ModuleAutoLoadSuccessMessage;
diff --git a/packages/sharing-editor/src/App.tsx b/packages/sharing-editor/src/App.tsx
index 8eee17b03..445f7b077 100644
--- a/packages/sharing-editor/src/App.tsx
+++ b/packages/sharing-editor/src/App.tsx
@@ -1,12 +1,18 @@
import React, { useEffect, useCallback, useRef } from "react";
import "./App.css";
-import { embedAppDataToUrl, AppData, File } from "@stlite/sharing-common";
+import {
+ embedAppDataToUrl,
+ AppData,
+ File,
+ BackwardMessage,
+} from "@stlite/sharing-common";
import { LoaderFunctionArgs, useLoaderData, redirect } from "react-router-dom";
import { useAppData } from "./use-app-data";
import StliteSharingIFrame, {
+ StliteSharingIFrameProps,
StliteSharingIFrameRef,
} from "./StliteSharingIFrame";
-import Editor, { EditorProps } from "./Editor";
+import Editor, { EditorProps, EditorRef } from "./Editor";
import PreviewToolBar from "./components/PreviewToolBar";
import { extractAppDataFromUrl } from "@stlite/sharing-common";
import {
@@ -83,6 +89,7 @@ function App() {
}, [initialAppData, initializeAppData]);
const iframeRef = useRef<StliteSharingIFrameRef>(null);
+ const editorRef = useRef<EditorRef>(null);
const handleFileWrite = useCallback<EditorProps["onFileWrite"]>(
(path, value) => {
@@ -208,6 +215,37 @@ function App() {
[updateAppData]
);
+ const handleIframeMessage = useCallback<
+ StliteSharingIFrameProps["onMessage"]
+ >(
+ (e) => {
+ if (e.data.stlite !== true) {
+ return;
+ }
+ const msg = e.data as BackwardMessage;
+ switch (msg.type) {
+ case "moduleAutoLoadSuccess": {
+ if (msg.data.loadedPackages.length > 0) {
+ const additionalRequirements = msg.data.packagesToLoad;
+ const editor = editorRef.current;
+ if (editor == null) {
+ return;
+ }
+ editor.addRequirements(
+ additionalRequirements.map((r) => r + " # auto-loaded")
+ );
+ updateAppData((cur) => ({
+ ...cur,
+ requirements: cur.requirements.concat(additionalRequirements),
+ }));
+ }
+ break;
+ }
+ }
+ },
+ [updateAppData]
+ );
+
return (
<div className="App">
{!embedMode && (
@@ -222,6 +260,7 @@ function App() {
left={
<Editor
key={initAppDataKey}
+ ref={editorRef}
appData={appData}
onFileWrite={handleFileWrite}
onFileRename={handleFileRename}
@@ -246,6 +285,7 @@ function App() {
messageTargetOrigin={SHARING_APP_ORIGIN}
title="stlite app"
className="preview-iframe"
+ onMessage={handleIframeMessage}
/>
)}
</>
diff --git a/packages/sharing-editor/src/Editor/components/Tab.module.scss b/packages/sharing-editor/src/Editor/components/Tab.module.scss
index a7d149b3b..96fab0704 100644
--- a/packages/sharing-editor/src/Editor/components/Tab.module.scss
+++ b/packages/sharing-editor/src/Editor/components/Tab.module.scss
@@ -22,7 +22,7 @@
border-top: var(--c-primary) var.$tab-highlight-height solid;
margin-top: -(var.$tab-highlight-height);
border-bottom: none;
- position: relative
+ position: relative;
}
$tabPaddingLeft: 0.5rem;
diff --git a/packages/sharing-editor/src/Editor/index.tsx b/packages/sharing-editor/src/Editor/index.tsx
index 9942c7c7f..0f6b912a8 100644
--- a/packages/sharing-editor/src/Editor/index.tsx
+++ b/packages/sharing-editor/src/Editor/index.tsx
@@ -4,6 +4,7 @@ import React, {
useCallback,
useRef,
useEffect,
+ useImperativeHandle,
} from "react";
import MonacoEditor, { OnMount } from "@monaco-editor/react";
import { AppData } from "@stlite/sharing-common";
@@ -25,6 +26,9 @@ let newFileCount = 1;
const REQUIREMENTS_FILENAME = "requirements";
+export interface EditorRef {
+ addRequirements: (requirements: string[]) => void;
+}
export interface EditorProps {
appData: AppData;
onFileWrite: (path: string, value: string | Uint8Array) => void;
@@ -33,239 +37,275 @@ export interface EditorProps {
onRequirementsChange: (requirements: string[]) => void;
}
-function Editor({
- appData,
- onFileWrite,
- onFileRename,
- onFileDelete,
- onRequirementsChange,
-}: EditorProps) {
- // Keep the tab order
- const [tabFileNames, setTabFileNames] = useState<string[]>(
- Object.keys(appData.files)
- );
-
- const fileNames = useMemo(
- () =>
- Object.keys(appData.files).sort((a, b) => {
- const aIdx = tabFileNames.indexOf(a);
- const bIdx = tabFileNames.indexOf(b);
-
- // If the key is not found in `tabFileNames` unexpectedly, put its tab at the right most.
- if (aIdx === -1) {
- return 1;
- }
- if (bIdx === -1) {
- return -1;
- }
-
- // Sort the keys of appData.files according to `tabFileNames`.
- return aIdx - bIdx;
- }),
- [appData, tabFileNames]
- );
- const [currentFileName, setCurrentFileName] = useState<string | null>(
- fileNames.length > 0 ? fileNames[0] : null
- );
+const Editor = React.forwardRef<EditorRef, EditorProps>(
+ (
+ { appData, onFileWrite, onFileRename, onFileDelete, onRequirementsChange },
+ ref
+ ) => {
+ // Keep the tab order
+ const [tabFileNames, setTabFileNames] = useState<string[]>(
+ Object.keys(appData.files)
+ );
- const currentFile =
- typeof currentFileName === "string" ? appData.files[currentFileName] : null;
+ const fileNames = useMemo(
+ () =>
+ Object.keys(appData.files).sort((a, b) => {
+ const aIdx = tabFileNames.indexOf(a);
+ const bIdx = tabFileNames.indexOf(b);
- const editorRef = useRef<Parameters<OnMount>[0]>(null);
- const monacoRef = useRef<any>(null);
- const handleEditorDitMount = useCallback<OnMount>((editor, monaco) => {
- editorRef.current = editor;
- monacoRef.current = monaco;
- }, []);
- useEffect(() => {
- return () => {
- const monaco = monacoRef.current;
- if (monaco) {
- // Clear all the existing models. Ref: https://stackoverflow.com/a/62466612/13103190
- // If we don't do it, the previous content will remain after changing the sample apps.
- // @ts-ignore
- monaco.editor.getModels().forEach((model) => model.dispose());
- }
- };
- }, []);
+ // If the key is not found in `tabFileNames` unexpectedly, put its tab at the right most.
+ if (aIdx === -1) {
+ return 1;
+ }
+ if (bIdx === -1) {
+ return -1;
+ }
- const handleSave = useCallback(() => {
- if (currentFileName == null) {
- return;
- }
- const editor = editorRef.current;
- if (editor == null) {
- return;
- }
+ // Sort the keys of appData.files according to `tabFileNames`.
+ return aIdx - bIdx;
+ }),
+ [appData, tabFileNames]
+ );
+ const [currentFileName, setCurrentFileName] = useState<string | null>(
+ fileNames.length > 0 ? fileNames[0] : null
+ );
- const value: string = editor.getValue();
- if (currentFileName === REQUIREMENTS_FILENAME) {
- const requirements = parseRequirementsTxt(value);
- onRequirementsChange(requirements);
- return;
- }
+ const currentFile =
+ typeof currentFileName === "string"
+ ? appData.files[currentFileName]
+ : null;
- onFileWrite(currentFileName, value);
- }, [onFileWrite, onRequirementsChange, currentFileName]);
+ const editorRef = useRef<Parameters<OnMount>[0]>(null);
+ const monacoRef = useRef<any>(null);
+ const handleEditorDitMount = useCallback<OnMount>((editor, monaco) => {
+ editorRef.current = editor;
+ monacoRef.current = monaco;
+ }, []);
+ useEffect(() => {
+ return () => {
+ const monaco = monacoRef.current;
+ if (monaco) {
+ // Clear all the existing models. Ref: https://stackoverflow.com/a/62466612/13103190
+ // If we don't do it, the previous content will remain after changing the sample apps.
+ // @ts-ignore
+ monaco.editor.getModels().forEach((model) => model.dispose());
+ }
+ };
+ }, []);
- const handleBinaryFileChange = useCallback(
- (data: Uint8Array) => {
+ const handleSave = useCallback(() => {
if (currentFileName == null) {
return;
}
- if (typeof currentFileName !== "string") {
+ const editor = editorRef.current;
+ if (editor == null) {
return;
}
- onFileWrite(currentFileName, data);
- },
- [onFileWrite, currentFileName]
- );
- const [addedFileName, setAddedFileName] = useState<string>();
- const focusTabNext = useCallback((fileName: string) => {
- setCurrentFileName(fileName);
- setAddedFileName(fileName);
- }, []);
+ const value: string = editor.getValue();
+ if (currentFileName === REQUIREMENTS_FILENAME) {
+ const requirements = parseRequirementsTxt(value);
+ onRequirementsChange(requirements);
+ return;
+ }
- const handleFileUpload = useCallback<FileUploaderProps["onUpload"]>(
- (files) => {
- files.forEach((file) => {
- if (file.type.startsWith("text")) {
- const text = new TextDecoder().decode(file.data);
- onFileWrite(file.name, text);
- focusTabNext(file.name);
- } else {
- onFileWrite(file.name, file.data);
- focusTabNext(file.name);
+ onFileWrite(currentFileName, value);
+ }, [onFileWrite, onRequirementsChange, currentFileName]);
+
+ const handleBinaryFileChange = useCallback(
+ (data: Uint8Array) => {
+ if (currentFileName == null) {
+ return;
+ }
+ if (typeof currentFileName !== "string") {
+ return;
}
- setTabFileNames((cur) => [...cur, file.name]);
- });
- },
- [onFileWrite, focusTabNext]
- );
+ onFileWrite(currentFileName, data);
+ },
+ [onFileWrite, currentFileName]
+ );
- const handleFileDelete = useCallback(
- (fileName: string) => {
- const confirmed = window.confirm(`Delete ${fileName}?`);
- if (!confirmed) {
- return;
- }
+ const [addedFileName, setAddedFileName] = useState<string>();
+ const focusTabNext = useCallback((fileName: string) => {
+ setCurrentFileName(fileName);
+ setAddedFileName(fileName);
+ }, []);
+
+ const handleFileUpload = useCallback<FileUploaderProps["onUpload"]>(
+ (files) => {
+ files.forEach((file) => {
+ if (file.type.startsWith("text")) {
+ const text = new TextDecoder().decode(file.data);
+ onFileWrite(file.name, text);
+ focusTabNext(file.name);
+ } else {
+ onFileWrite(file.name, file.data);
+ focusTabNext(file.name);
+ }
+ setTabFileNames((cur) => [...cur, file.name]);
+ });
+ },
+ [onFileWrite, focusTabNext]
+ );
- onFileDelete(fileName);
- setTabFileNames((cur) => cur.filter((f) => f !== fileName));
- },
- [onFileDelete]
- );
+ const handleFileDelete = useCallback(
+ (fileName: string) => {
+ const confirmed = window.confirm(`Delete ${fileName}?`);
+ if (!confirmed) {
+ return;
+ }
- const handleCreateFile = useCallback(() => {
- const fileName = `file${newFileCount}.py`;
- newFileCount += 1;
+ onFileDelete(fileName);
+ setTabFileNames((cur) => cur.filter((f) => f !== fileName));
+ },
+ [onFileDelete]
+ );
- onFileWrite(fileName, "");
- focusTabNext(fileName);
- setTabFileNames((cur) => [...cur, fileName]);
- }, [onFileWrite, focusTabNext]);
+ const handleCreateFile = useCallback(() => {
+ const fileName = `file${newFileCount}.py`;
+ newFileCount += 1;
- const showTextEditor =
- currentFile?.content?.$case === "text" ||
- currentFileName === REQUIREMENTS_FILENAME;
+ onFileWrite(fileName, "");
+ focusTabNext(fileName);
+ setTabFileNames((cur) => [...cur, fileName]);
+ }, [onFileWrite, focusTabNext]);
- const defaultRequirementsTextValue = useMemo(
- () => appData.requirements.join("\n"),
- [appData.requirements]
- );
+ const showTextEditor =
+ currentFile?.content?.$case === "text" ||
+ currentFileName === REQUIREMENTS_FILENAME;
- const [isDarkTheme, setIsDarkTheme] = useLocalStorage(
- "editor-theme",
- isDarkMode()
- );
+ const defaultRequirementsTextValue = useMemo(
+ () => appData.requirements.join("\n"),
+ [appData.requirements]
+ );
- return (
- <div className={styles.container}>
- <ResizableHeader
- resizableArea={
- <TabBar>
- {fileNames.map((fileName) => (
- <Tab
- key={fileName}
- selected={fileName === currentFileName}
- fileNameEditable={fileName !== appData.entrypoint}
- initInEditingModeIfSelected={fileName === addedFileName}
- fileName={fileName}
- onSelect={() => setCurrentFileName(fileName)}
- onDelete={() => handleFileDelete(fileName)}
- onFileNameChange={(newPath) => {
- onFileRename(fileName, newPath);
- setTabFileNames((cur) =>
- cur.map((f) => (f === fileName ? newPath : f))
- );
- if (fileName === currentFileName) {
- setCurrentFileName(newPath);
- }
- }}
- />
- ))}
- <div className={styles.controlButtonGroup}>
- <AddButton onClick={handleCreateFile} />
- <FileUploader onUpload={handleFileUpload} />
- </div>
+ const [isDarkTheme, setIsDarkTheme] = useLocalStorage(
+ "editor-theme",
+ isDarkMode()
+ );
+
+ useImperativeHandle(
+ ref,
+ () => ({
+ addRequirements: (additionalRequirements) => {
+ const monaco = monacoRef.current;
+ if (monaco == null) {
+ return;
+ }
+
+ // Handle Monaco Editor's model directly imperatively to update the value. Ref: https://github.com/suren-atoyan/monaco-react/blob/v4.6.0/src/utils/index.ts#L21
+ const uri = monaco.Uri.parse(REQUIREMENTS_FILENAME);
+ const model =
+ monaco.editor.getModel(uri) ??
+ monaco.editor.createModel(
+ defaultRequirementsTextValue,
+ "text",
+ uri
+ );
+
+ const curValue = model.getValue();
+ const newLineNeeded = curValue !== "" && !curValue.endsWith("\n");
+ const newValue =
+ curValue +
+ (newLineNeeded ? "\n" : "") +
+ additionalRequirements.join("\n");
- <div className={styles.requirementsTabContainer}>
- <Tab
- selected={currentFileName === REQUIREMENTS_FILENAME}
- fileNameEditable={false}
- initInEditingModeIfSelected={false}
- fileName={REQUIREMENTS_FILENAME}
- onSelect={() => setCurrentFileName(REQUIREMENTS_FILENAME)}
- onDelete={() => null}
- onFileNameChange={() => null}
+ model.setValue(newValue);
+ },
+ }),
+ [defaultRequirementsTextValue]
+ );
+
+ return (
+ <div className={styles.container}>
+ <ResizableHeader
+ resizableArea={
+ <TabBar>
+ {fileNames.map((fileName) => (
+ <Tab
+ key={fileName}
+ selected={fileName === currentFileName}
+ fileNameEditable={fileName !== appData.entrypoint}
+ initInEditingModeIfSelected={fileName === addedFileName}
+ fileName={fileName}
+ onSelect={() => setCurrentFileName(fileName)}
+ onDelete={() => handleFileDelete(fileName)}
+ onFileNameChange={(newPath) => {
+ onFileRename(fileName, newPath);
+ setTabFileNames((cur) =>
+ cur.map((f) => (f === fileName ? newPath : f))
+ );
+ if (fileName === currentFileName) {
+ setCurrentFileName(newPath);
+ }
+ }}
+ />
+ ))}
+ <div className={styles.controlButtonGroup}>
+ <AddButton onClick={handleCreateFile} />
+ <FileUploader onUpload={handleFileUpload} />
+ </div>
+
+ <div className={styles.requirementsTabContainer}>
+ <Tab
+ selected={currentFileName === REQUIREMENTS_FILENAME}
+ fileNameEditable={false}
+ initInEditingModeIfSelected={false}
+ fileName={REQUIREMENTS_FILENAME}
+ onSelect={() => setCurrentFileName(REQUIREMENTS_FILENAME)}
+ onDelete={() => null}
+ onFileNameChange={() => null}
+ />
+ </div>
+ </TabBar>
+ }
+ fixedFooter={
+ showTextEditor && (
+ <Toolbar>
+ <SaveButton onClick={handleSave} />
+ <ThemeSelect isDark={isDarkTheme} onChange={setIsDarkTheme} />
+ </Toolbar>
+ )
+ }
+ />
+ <div className={styles.editorArea}>
+ <div
+ // NOTE: Keep the monaco-editor component being mounted
+ // and control its visibility with the hidden attribute here
+ // instead of mounting/unmounting the component according to the file type
+ // because it leads to flickering.
+ hidden={!showTextEditor}
+ style={{ height: "100%" }}
+ >
+ <MonacoEditor
+ path={
+ typeof currentFileName === "string"
+ ? currentFileName
+ : undefined
+ }
+ defaultValue={
+ currentFileName === REQUIREMENTS_FILENAME
+ ? defaultRequirementsTextValue
+ : currentFile?.content?.$case === "text"
+ ? currentFile.content.text
+ : undefined
+ }
+ onMount={handleEditorDitMount}
+ theme={isDarkTheme ? "vs-dark" : "vs"}
+ />
+ </div>
+ {currentFileName != null &&
+ currentFile?.content?.$case === "data" && (
+ <BinaryFileEditor
+ path={currentFileName}
+ data={currentFile.content.data}
+ onChange={handleBinaryFileChange}
/>
- </div>
- </TabBar>
- }
- fixedFooter={
- showTextEditor && (
- <Toolbar>
- <SaveButton onClick={handleSave} />
- <ThemeSelect isDark={isDarkTheme} onChange={setIsDarkTheme} />
- </Toolbar>
- )
- }
- />
- <div className={styles.editorArea}>
- <div
- // NOTE: Keep the monaco-editor component being mounted
- // and control its visibility with the hidden attribute here
- // instead of mounting/unmounting the component according to the file type
- // because it leads to flickering.
- hidden={!showTextEditor}
- style={{ height: "100%" }}
- >
- <MonacoEditor
- path={
- typeof currentFileName === "string" ? currentFileName : undefined
- }
- defaultValue={
- currentFileName === REQUIREMENTS_FILENAME
- ? defaultRequirementsTextValue
- : currentFile?.content?.$case === "text"
- ? currentFile.content.text
- : undefined
- }
- onMount={handleEditorDitMount}
- theme={isDarkTheme ? "vs-dark" : "light"}
- />
+ )}
</div>
- {currentFileName != null && currentFile?.content?.$case === "data" && (
- <BinaryFileEditor
- path={currentFileName}
- data={currentFile.content.data}
- onChange={handleBinaryFileChange}
- />
- )}
</div>
- </div>
- );
-}
+ );
+ }
+);
export default Editor;
diff --git a/packages/sharing-editor/src/StliteSharingIFrame/index.tsx b/packages/sharing-editor/src/StliteSharingIFrame/index.tsx
index e6722d445..a6dc89985 100644
--- a/packages/sharing-editor/src/StliteSharingIFrame/index.tsx
+++ b/packages/sharing-editor/src/StliteSharingIFrame/index.tsx
@@ -1,4 +1,4 @@
-import React, { useRef, useMemo, useImperativeHandle } from "react";
+import React, { useRef, useMemo, useImperativeHandle, useEffect } from "react";
import {
AppData,
embedAppDataToUrl,
@@ -15,13 +15,20 @@ export interface StliteSharingIFrameProps extends Omit<IFrameProps, "src"> {
sharingAppSrc: string;
initialAppData: AppData;
messageTargetOrigin: string;
+ onMessage: (event: MessageEvent) => void;
}
const StliteSharingIFrame = React.forwardRef<
StliteSharingIFrameRef,
StliteSharingIFrameProps
>(
(
- { sharingAppSrc, initialAppData, messageTargetOrigin, ...iframeProps },
+ {
+ sharingAppSrc,
+ initialAppData,
+ messageTargetOrigin,
+ onMessage,
+ ...iframeProps
+ },
ref
) => {
const iframeRef = useRef<HTMLIFrameElement>(null);
@@ -66,6 +73,19 @@ const StliteSharingIFrame = React.forwardRef<
[messageTargetOrigin]
);
+ useEffect(() => {
+ const windowMessageEventListener = (event: MessageEvent) => {
+ if (event.source === iframeRef.current?.contentWindow) {
+ onMessage(event);
+ }
+ };
+
+ window.addEventListener("message", windowMessageEventListener);
+ return () => {
+ window.removeEventListener("message", windowMessageEventListener);
+ };
+ }, [onMessage]);
+
return (
// eslint-disable-next-line jsx-a11y/iframe-has-title
<iframe
diff --git a/packages/sharing/src/App.tsx b/packages/sharing/src/App.tsx
index 41ba5a7c2..be5d068a1 100644
--- a/packages/sharing/src/App.tsx
+++ b/packages/sharing/src/App.tsx
@@ -5,6 +5,7 @@ import {
extractAppDataFromUrl,
ForwardMessage,
ReplyMessage,
+ ModuleAutoLoadSuccessMessage,
} from "@stlite/sharing-common";
import StreamlitApp from "./StreamlitApp";
import {
@@ -24,6 +25,8 @@ function isEditorOrigin(origin: string): boolean {
return origin === process.env.REACT_APP_EDITOR_APP_ORIGIN;
}
+let communicatedEditorOrigin = "";
+
function convertFiles(
appDataFiles: AppData["files"]
): StliteKernelOptions["files"] {
@@ -83,11 +86,35 @@ st.write("Hello World")`,
requirements: appData.requirements,
prebuiltPackageNames: [],
...makeToastKernelCallbacks(),
+ moduleAutoLoad: true,
});
_kernel = kernel;
setKernel(kernel);
- const kernelWithToast = new StliteKernelWithToast(kernel);
+ const kernelWithToast = new StliteKernelWithToast(kernel, {
+ onModuleAutoLoad: (packagesToLoad, installPromise) => {
+ console.log("Module auto-load started", packagesToLoad);
+ installPromise
+ .then((loadedPackages) => {
+ console.log("Module auto-load success", loadedPackages);
+ window.parent.postMessage(
+ {
+ type: "moduleAutoLoadSuccess",
+ data: {
+ packagesToLoad,
+ loadedPackages,
+ },
+ stlite: true,
+ } as ModuleAutoLoadSuccessMessage,
+ process.env.REACT_APP_EDITOR_APP_ORIGIN ??
+ communicatedEditorOrigin // Fall back to the origin of the last message from the editor app if the REACT_APP_EDITOR_APP_ORIGIN env var is not set, i.e. in preview deployments.
+ );
+ })
+ .catch((error) => {
+ console.error("Auto install failed", error);
+ });
+ },
+ });
// Handle messages from the editor
onMessage = (event) => {
@@ -95,6 +122,8 @@ st.write("Hello World")`,
return;
}
+ communicatedEditorOrigin = event.origin;
+
const port2 = event.ports[0];
function postReplyMessage(msg: ReplyMessage) {
port2.postMessage(msg);
diff --git a/streamlit b/streamlit
index 026039e5d..f76a269dc 160000
--- a/streamlit
+++ b/streamlit
@@ -1 +1 @@
-Subproject commit 026039e5dff8f4c3d37e0c6b6932ff177a20d800
+Subproject commit f76a269dc4695f0e4724c6a6c52bbfb3e8b99f86