Lite: filesystem and requirements installer (#4779)

.changeset/proud-ears-teach.md

@@ -0,0 +1,5 @@
+---
+"@gradio/lite": patch
+---
+
+Add file system APIs and an imperative package install method

js/app/lite.html

@@ -24,8 +24,9 @@
 		<textarea id="code-input" cols="30" rows="10">
 import gradio as gr
 
-def greet(name):
-    return "Hello " + name + "!"
+from greetings import greet
+# def greet(name):
+#     return "Hello " + name + "!"
 
 def upload_file(files):
     file_paths = [file.name for file in files]
@@ -47,14 +48,38 @@ demo.launch()
 		</textarea>
 		<button id="exec-button">Execute</button>
 
+		<textarea id="requirements-input"></textarea>
+		<button id="install-button">Install</button>
+
 		<script type="module">  // type="module" is necessary to use `createGradioApp()`, which is loaded with <script type="module" /> tag above.
 			const code_input = document.getElementById("code-input");
 			const exec_button = document.getElementById("exec-button");
+			const requirements_input = document.getElementById("requirements-input");
+			const install_button = document.getElementById("install-button");
+
+			function parse_requirements(text) {
+				return text
+					.split("\n")
+					.map((line) => line.trim())
+					.filter((line) => line.length > 0 && !line.startsWith("#"));
+			}
 
 			const initial_code = code_input.value;
+			const initial_requirements = parse_requirements(requirements_input.value);
 
 			const controller = createGradioApp({
 				target: document.getElementById("gradio-app"),
+				files: {
+					"greetings.py": {
+						data: `
+def greet(name):
+    return "Hello " + name + "!"
+`
+					},
+					"images/logo.png": {
+						url: "https://raw.githubusercontent.com/gradio-app/gradio/main/guides/assets/logo.png"
+					}
+				},
 				pyCode: initial_code,
 				info: true,
 				container: true,
@@ -70,6 +95,15 @@ demo.launch()
 			exec_button.onclick = () => {
 				console.debug("exec_button.onclick");
 				controller.rerun(code_input.value);
+				console.debug("Rerun finished")
+			}
+
+			install_button.onclick = async () => {
+				console.debug("install_button.onclick");
+				const requirements = parse_requirements(requirements_input.value)
+				console.debug("requirements", requirements)
+				controller.install(requirements);
+				console.debug("Install finished")
 			}
 		</script>
 	</body>

js/app/src/lite/index.ts

@@ -1,5 +1,5 @@
 import "@gradio/theme";
-import { WorkerProxy } from "@gradio/wasm";
+import { WorkerProxy, type WorkerProxyOptions } from "@gradio/wasm";
 import { api_factory } from "@gradio/client";
 import { wasm_proxied_fetch } from "./fetch";
 import { wasm_proxied_mount_css } from "./css";
@@ -30,10 +30,16 @@ declare let GRADIO_VERSION: string;
 
 interface GradioAppController {
 	rerun: (code: string) => Promise<void>;
+	write: (path: string, data: string | ArrayBufferView, opts: any) => Promise<void>;
+	rename: (old_path: string, new_path: string) => Promise<void>;
+	unlink: (path: string) => Promise<void>;
+	install: (requirements: string[]) => Promise<void>;
 }
 
 interface Options {
 	target: HTMLElement;
+	files?: WorkerProxyOptions["files"];
+	requirements?: WorkerProxyOptions["requirements"];
 	pyCode: string;
 	info: boolean;
 	container: boolean;
@@ -57,7 +63,8 @@ export function create(options: Options): GradioAppController {
 	const worker_proxy = new WorkerProxy({
 		gradioWheelUrl: new URL(gradioWheel, import.meta.url).href,
 		gradioClientWheelUrl: new URL(gradioClientWheel, import.meta.url).href,
-		requirements: []
+		files: options.files ?? {},
+		requirements: options.requirements ?? [],
 	});
 
 	// Internally, the execution of `runPythonAsync()` is queued
@@ -116,6 +123,18 @@ export function create(options: Options): GradioAppController {
 		rerun: async (code: string): Promise<void> => {
 			await worker_proxy.runPythonAsync(code);
 			launchNewApp();
+		},
+		write(path, data, opts) {
+			return worker_proxy.writeFile(path, data, opts);
+		},
+		rename(old_path: string, new_path: string): Promise<void> {
+			return worker_proxy.renameFile(old_path, new_path);
+		},
+		unlink(path) {
+			return worker_proxy.unlink(path);
+		},
+		install(requirements) {
+			return worker_proxy.install(requirements);
 		}
 	};
 }

js/wasm/package.json

@@ -22,5 +22,9 @@
 	},
 	"devDependencies": {
 		"pyodide": "^0.23.2"
+	},
+	"dependencies": {
+		"@types/path-browserify": "^1.0.0",
+		"path-browserify": "^1.0.1"
 	}
 }

js/wasm/src/index.ts

@@ -1 +1 @@
-export { WorkerProxy } from "./worker-proxy";
+export { WorkerProxy, type WorkerProxyOptions } from "./worker-proxy";

js/wasm/src/message-types.ts

@@ -5,12 +5,19 @@ export interface HttpRequest {
 	headers: Record<string, string>;
 	body?: Uint8Array;
 }
-
 export interface HttpResponse {
 	status: number;
 	headers: Record<string, string>;
 	body: Uint8Array;
 }
+export interface EmscriptenFile {
+	data: string | ArrayBufferView;
+	opts?: Record<string, string>;
+}
+export interface EmscriptenFileUrl {
+	url: string;
+	opts?: Record<string, string>;
+}
 
 export interface InMessageBase {
 	type: string;
@@ -22,6 +29,7 @@ export interface InMessageInit extends InMessageBase {
 	data: {
 		gradioWheelUrl: string;
 		gradioClientWheelUrl: string;
+		files: Record<string, EmscriptenFile | EmscriptenFileUrl>;
 		requirements: string[];
 	};
 }
@@ -37,6 +45,33 @@ export interface InMessageHttpRequest extends InMessageBase {
 		request: HttpRequest;
 	};
 }
+export interface InMessageFileWrite extends InMessageBase {
+	type: "file:write";
+	data: {
+		path: string;
+		data: string | ArrayBufferView;
+		opts?: Record<string, any>;
+	};
+}
+export interface InMessageFileRename extends InMessageBase {
+	type: "file:rename";
+	data: {
+		oldPath: string;
+		newPath: string;
+	};
+}
+export interface InMessageFileUnlink extends InMessageBase {
+	type: "file:unlink";
+	data: {
+		path: string;
+	};
+}
+export interface InMessageInstall extends InMessageBase {
+	type: "install";
+	data: {
+		requirements: string[];
+	};
+}
 
 export interface InMessageEcho extends InMessageBase {
 	// For debug
@@ -48,6 +83,10 @@ export type InMessage =
 	| InMessageInit
 	| InMessageRunPython
 	| InMessageHttpRequest
+	| InMessageFileWrite
+	| InMessageFileRename
+	| InMessageFileUnlink
+	| InMessageInstall
 	| InMessageEcho;
 
 export interface ReplyMessageSuccess<T = unknown> {

js/wasm/src/webworker/file.test.ts

@@ -0,0 +1,90 @@
+// @vitest-environment node
+
+import path from "path";
+import { loadPyodide, PyodideInterface } from "pyodide";
+import { describe, it, expect, beforeEach } from "vitest";
+import { writeFileWithParents, renameWithParents } from "./file";
+
+describe("writeFileWithParents()", () => {
+	let pyodide: PyodideInterface;
+
+	beforeEach(async () => {
+		pyodide = await loadPyodide({
+			indexURL: path.resolve(__dirname, "../../node_modules/pyodide"),
+		});
+	});
+
+	const testCases: { paths: string[] }[] = [
+		{ paths: ["foo.py"] },
+		{ paths: ["foo/bar.py"] },
+		{ paths: ["foo/bar.py", "foo/hoge.py"] },
+		{ paths: ["foo/bar/baz.py"] },
+		{ paths: ["foo/bar/baz.py", "foo/bar/hoge.py"] },
+		{ paths: ["/foo.py"] },
+	];
+	testCases.forEach(({ paths }) => {
+		it(`writes files (${paths})`, () => {
+			for (const path of paths) {
+				expect(pyodide.FS.analyzePath(path).exists).toBe(false);
+
+				writeFileWithParents(pyodide, path, "# Test");
+
+				expect(pyodide.FS.analyzePath(path).exists).toBe(true);
+				expect(pyodide.FS.readFile(path, { encoding: "utf8" })).toEqual(
+					"# Test"
+				);
+			}
+		});
+	});
+
+	it("can write binary files", () => {
+		const path = "foo/bar.dat";
+		const uint8View = new Uint8Array([0, 1, 2, 3]); // Random data
+		writeFileWithParents(pyodide, path, uint8View);
+		expect(pyodide.FS.readFile(path)).toEqual(uint8View);
+	});
+});
+
+describe("renameWithParents", () => {
+	let pyodide: PyodideInterface;
+
+	beforeEach(async () => {
+		pyodide = await loadPyodide({
+			indexURL: path.resolve(__dirname, "../../node_modules/pyodide"),
+		});
+	});
+
+	const testCases: { oldPath: string; newPath: string }[] = [
+		{ oldPath: "foo.py", newPath: "bar.py" }, // Same dir, without a parent path
+		{ oldPath: "foo.py", newPath: "bar/baz.py" }, // To a nested dir
+		{ oldPath: "baz/foo.py", newPath: "bar.py" }, // From a nested dir
+		{ oldPath: "foo/bar.py", newPath: "foo/baz.py" }, // Same dir with a parent path
+		{ oldPath: "foo/bar.py", newPath: "baz/qux.py" }, // With parent paths, different dirs
+	];
+	testCases.forEach(({ oldPath, newPath }) => {
+		it(`renames "${oldPath}" to "${newPath}"`, () => {
+			writeFileWithParents(pyodide, oldPath, "# Test");
+			expect(pyodide.FS.analyzePath(oldPath).exists).toBe(true);
+
+			renameWithParents(pyodide, oldPath, newPath);
+
+			expect(pyodide.FS.analyzePath(oldPath).exists).toBe(false);
+			expect(pyodide.FS.analyzePath(newPath).exists).toBe(true);
+			expect(pyodide.FS.readFile(newPath, { encoding: "utf8" })).toEqual(
+				"# Test"
+			);
+		});
+	});
+
+	["foo.py", "foo/bar.py"].forEach((path) => {
+		it(`does nothing when the source and the destination are the same`, () => {
+			writeFileWithParents(pyodide, path, "# Test");
+			expect(pyodide.FS.analyzePath(path).exists).toBe(true);
+
+			renameWithParents(pyodide, path, path);
+
+			expect(pyodide.FS.analyzePath(path).exists).toBe(true);
+			expect(pyodide.FS.readFile(path, { encoding: "utf8" })).toEqual("# Test");
+		});
+	});
+});

js/wasm/src/webworker/file.ts

@@ -0,0 +1,49 @@
+import path from "path-browserify";
+import type { PyodideInterface } from "pyodide";
+
+function ensureParent(pyodide: PyodideInterface, filePath: string): void {
+	const normalized = path.normalize(filePath);
+
+	const dirPath = path.dirname(normalized);
+
+	const dirNames = dirPath.split("/");
+
+	const chDirNames: string[] = [];
+	for (const dirName of dirNames) {
+		chDirNames.push(dirName);
+		const dirPath = chDirNames.join("/");
+
+		if (pyodide.FS.analyzePath(dirPath).exists) {
+			if (pyodide.FS.isDir(dirPath)) {
+				throw new Error(`"${dirPath}" already exists and is not a directory.`);
+			}
+			continue;
+		}
+
+		try {
+			pyodide.FS.mkdir(dirPath);
+		} catch (err) {
+			console.error(`Failed to create a directory "${dirPath}"`);
+			throw err;
+		}
+	}
+}
+
+export function writeFileWithParents(
+	pyodide: PyodideInterface,
+	filePath: string,
+	data: string | ArrayBufferView,
+	opts?: Parameters<PyodideInterface["FS"]["writeFile"]>[2]
+): void {
+	ensureParent(pyodide, filePath);
+	pyodide.FS.writeFile(filePath, data, opts);
+}
+
+export function renameWithParents(
+	pyodide: PyodideInterface,
+	oldPath: string,
+	newPath: string
+): void {
+	ensureParent(pyodide, newPath);
+	pyodide.FS.rename(oldPath, newPath);
+}

js/wasm/src/webworker/index.ts

@@ -1,11 +1,15 @@
 /// <reference lib="webworker" />
+/* eslint-env worker */
 
 import type { PyodideInterface } from "pyodide";
 import type {
 	InMessage,
+	InMessageInit,
 	ReplyMessageError,
 	ReplyMessageSuccess
 } from "../message-types";
+import { writeFileWithParents, renameWithParents } from "./file";
+import { verifyRequirements } from "./requirements";
 import { makeHttpRequest } from "./http";
 
 importScripts("https://cdn.jsdelivr.net/pyodide/v0.23.2/full/pyodide.js");
@@ -16,16 +20,11 @@ let pyodideReadyPromise: undefined | Promise<void> = undefined;
 
 let call_asgi_app_from_js: (
 	scope: unknown,
-	receive: Function,
-	send: Function
+	receive: () => Promise<unknown>,
+	send: (event: any) => Promise<void>
 ) => Promise<void>;
 
-interface InitOptions {
-	gradioWheelUrl: string;
-	gradioClientWheelUrl: string;
-	requirements: string[];
-}
-async function loadPyodideAndPackages(options: InitOptions) {
+async function loadPyodideAndPackages(options: InMessageInit["data"]): Promise<void> {
 	console.debug("Loading Pyodide.");
 	pyodide = await loadPyodide({
 		stdout: console.log,
@@ -33,6 +32,28 @@ async function loadPyodideAndPackages(options: InitOptions) {
 	});
 	console.debug("Pyodide is loaded.");
 
+	console.debug("Mounting files.", options.files);
+	await Promise.all(
+		Object.keys(options.files).map(async (path) => {
+			const file = options.files[path];
+
+			let data: string | ArrayBufferView;
+			if ("url" in file) {
+				console.debug(`Fetch a file from ${file.url}`);
+				data = await fetch(file.url)
+					.then((res) => res.arrayBuffer())
+					.then((buffer) => new Uint8Array(buffer));
+			} else {
+				data = file.data;
+			}
+			const { opts } = options.files[path];
+
+			console.debug(`Write a file "${path}"`);
+			writeFileWithParents(pyodide, path, data, opts);
+		})
+	);
+	console.debug("Files are mounted.");
+
 	console.debug("Loading micropip");
 	await pyodide.loadPackage("micropip");
 	const micropip = pyodide.pyimport("micropip");
@@ -137,7 +158,7 @@ matplotlib.use("agg")
 	console.debug("matplotlib backend is set.");
 }
 
-self.onmessage = async (event: MessageEvent<InMessage>) => {
+self.onmessage = async (event: MessageEvent<InMessage>): Promise<void> => {
 	const msg = event.data;
 	console.debug("worker.onmessage", msg);
 
@@ -145,17 +166,14 @@ self.onmessage = async (event: MessageEvent<InMessage>) => {
 
 	try {
 		if (msg.type === "init") {
-			pyodideReadyPromise = loadPyodideAndPackages({
-				gradioWheelUrl: msg.data.gradioWheelUrl,
-				gradioClientWheelUrl: msg.data.gradioClientWheelUrl,
-				requirements: msg.data.requirements
-			});
+			pyodideReadyPromise = loadPyodideAndPackages(msg.data);
 
 			const replyMessage: ReplyMessageSuccess = {
 				type: "reply:success",
 				data: null
 			};
 			messagePort.postMessage(replyMessage);
+			return;
 		}
 
 		if (pyodideReadyPromise == null) {
@@ -194,6 +212,72 @@ self.onmessage = async (event: MessageEvent<InMessage>) => {
 				messagePort.postMessage(replyMessage);
 				break;
 			}
+			case "file:write": {
+				const { path, data: fileData, opts } = msg.data;
+
+				console.debug(`Write a file "${path}"`);
+				writeFileWithParents(pyodide, path, fileData, opts);
+
+				const replyMessage: ReplyMessageSuccess = {
+					type: "reply:success",
+					data: null
+				};
+				messagePort.postMessage(replyMessage);
+				break;
+			}
+			case "file:rename": {
+				const { oldPath, newPath } = msg.data;
+
+				console.debug(`Rename "${oldPath}" to ${newPath}`);
+				renameWithParents(pyodide, oldPath, newPath);
+
+				const replyMessage: ReplyMessageSuccess = {
+					type: "reply:success",
+					data: null
+				};
+				messagePort.postMessage(replyMessage);
+				break;
+			}
+			case "file:unlink": {
+				const { path } = msg.data;
+
+				console.debug(`Remove "${path}`);
+				pyodide.FS.unlink(path);
+
+				const replyMessage: ReplyMessageSuccess = {
+					type: "reply:success",
+					data: null
+				};
+				messagePort.postMessage(replyMessage);
+				break;
+			}
+			case "install": {
+				const { requirements } = msg.data;
+
+				const micropip = pyodide.pyimport("micropip");
+
+				console.debug("Install the requirements:", requirements);
+				verifyRequirements(requirements); // Blocks the not allowed wheel URL schemes.
+				await micropip.install
+					.callKwargs(requirements, { keep_going: true })
+					.then(() => {
+						if (requirements.includes("matplotlib")) {
+							return pyodide.runPythonAsync(`
+                from stlite_server.bootstrap import _fix_matplotlib_crash
+                _fix_matplotlib_crash()
+              `);
+						}
+					})
+					.then(() => {
+						console.debug("Successfully installed");
+
+						const replyMessage: ReplyMessageSuccess = {
+							type: "reply:success",
+							data: null,
+						};
+						messagePort.postMessage(replyMessage);
+					});
+			}
 		}
 	} catch (error) {
 		const replyMessage: ReplyMessageError = {

js/wasm/src/webworker/requirements.test.ts

@@ -0,0 +1,29 @@
+import { describe, it, expect } from "vitest";
+import { verifyRequirements } from "./requirements";
+
+describe("verifyRequirements", () => {
+	const allowedRequirements = [
+		[
+			"http://files.pythonhosted.org/packages/62/9c/0467dea0a064a998f94c33d03988f33efc744de1a2a550b56b38910cafa2/streamlit-1.13.0-py2.py3-none-any.whl",
+		],
+		[
+			"https://files.pythonhosted.org/packages/62/9c/0467dea0a064a998f94c33d03988f33efc744de1a2a550b56b38910cafa2/streamlit-1.13.0-py2.py3-none-any.whl",
+		],
+	];
+	allowedRequirements.forEach((requirements) => {
+		it(`allows http: and https: schemes (requirements=${JSON.stringify(
+			requirements
+		)})`, () => {
+			expect(() => verifyRequirements(requirements)).not.toThrow();
+		});
+	});
+
+	const notAllowedRequirements = [["emfs:/tmp/foo.whl"], ["file:/tmp/foo.whl"]];
+	notAllowedRequirements.forEach((requirements) => {
+		it(`throws an error if the requirements include a not allowed scheme (requirements=${JSON.stringify(
+			requirements
+		)})`, () => {
+			expect(() => verifyRequirements(requirements)).toThrow();
+		});
+	});
+});

js/wasm/src/webworker/requirements.ts

@@ -0,0 +1,18 @@
+export function verifyRequirements(requirements: string[]) {
+	requirements.forEach((req) => {
+		let url: URL;
+		try {
+			url = new URL(req);
+		} catch {
+			// `req` is not a URL -> OK
+			return;
+		}
+
+		// Ref: The scheme checker in the micropip implementation is https://github.com/pyodide/micropip/blob/v0.1.0/micropip/_compat_in_pyodide.py#L23-L26
+		if (url.protocol === "emfs:" || url.protocol === "file:") {
+			throw new Error(
+				`"emfs:" and "file:" protocols are not allowed for the requirement (${req})`
+			);
+		}
+	});
+}

js/wasm/src/worker-proxy.ts

@@ -1,5 +1,7 @@
 import { CrossOriginWorkerMaker as Worker } from "./cross-origin-worker";
 import type {
+	EmscriptenFile,
+	EmscriptenFileUrl,
 	HttpRequest,
 	HttpResponse,
 	InMessage,
@@ -9,6 +11,7 @@ import type {
 export interface WorkerProxyOptions {
 	gradioWheelUrl: string;
 	gradioClientWheelUrl: string;
+	files: Record<string, EmscriptenFile | EmscriptenFileUrl>;
 	requirements: string[];
 }
 
@@ -29,6 +32,7 @@ export class WorkerProxy {
 			data: {
 				gradioWheelUrl: options.gradioWheelUrl,
 				gradioClientWheelUrl: options.gradioClientWheelUrl,
+				files: options.files,
 				requirements: options.requirements
 			}
 		}).then(() => {
@@ -102,6 +106,49 @@ export class WorkerProxy {
 		return response;
 	}
 
+	public writeFile(
+		path: string,
+		data: string | ArrayBufferView,
+		opts?: Record<string, unknown>
+	): Promise<void> {
+		return this.postMessageAsync({
+			type: "file:write",
+			data: {
+				path,
+				data,
+				opts,
+			},
+		}) as Promise<void>;
+	}
+
+	public renameFile(oldPath: string, newPath: string): Promise<void> {
+		return this.postMessageAsync({
+			type: "file:rename",
+			data: {
+				oldPath,
+				newPath,
+			},
+		}) as Promise<void>;
+	}
+
+	public unlink(path: string): Promise<void> {
+		return this.postMessageAsync({
+			type: "file:unlink",
+			data: {
+				path,
+			},
+		}) as Promise<void>;
+	}
+
+	public install(requirements: string[]): Promise<void> {
+		return this.postMessageAsync({
+			type: "install",
+			data: {
+				requirements,
+			},
+		}) as Promise<void>;
+	}
+
 	public terminate(): void {
 		this.worker.terminate();
 	}

pnpm-lock.yaml

@@ -1,4 +1,4 @@
-lockfileVersion: '6.0'
+lockfileVersion: '6.1'
 
 settings:
   autoInstallPeers: true
@@ -777,6 +777,13 @@ importers:
         version: link:../upload
 
   js/wasm:
+    dependencies:
+      '@types/path-browserify':
+        specifier: ^1.0.0
+        version: 1.0.0
+      path-browserify:
+        specifier: ^1.0.1
+        version: 1.0.1
     devDependencies:
       pyodide:
         specifier: ^0.23.2
@@ -5953,6 +5960,10 @@ packages:
     resolution: {integrity: sha512-WKG4gTr8przEZBiJ5r3s8ZIAoMXNbOgQ+j/d5O4X3x6kZJRLNvyUJuUK/KoG3+8BaOHPhp2m7WC6JKKeovDSzQ==}
     dev: true
 
+  /@types/path-browserify@1.0.0:
+    resolution: {integrity: sha512-XMCcyhSvxcch8b7rZAtFAaierBYdeHXVvg2iYnxOV0MCQHmPuRRmGZPFDRzPayxcGiiSL1Te9UIO+f3cuj0tfw==}
+    dev: false
+
   /@types/pretty-hrtime@1.0.1:
     resolution: {integrity: sha512-VjID5MJb1eGKthz2qUerWT8+R4b9N+CHvGCzg9fn4kWZgaF9AhdYikQio3R7wV8YY1NsQKPaCwKz1Yff+aHNUQ==}
     dev: true
@@ -10935,6 +10946,10 @@ packages:
     engines: {node: '>= 0.8'}
     dev: true
 
+  /path-browserify@1.0.1:
+    resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==}
+    dev: false
+
   /path-exists@3.0.0:
     resolution: {integrity: sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==}
     engines: {node: '>=4'}