E2E tests for Lite (#6890)

* Set up E2E test config for lite

* Use the same Page instance for all the tests in the case of lite

* Fix reading demo files

* Fix config

* Install requirements based on `requirements.txt`

* Add the "loaded" event dispatched from the main component to make a promise wait for the compoonent to be loaded

* Refactor js/tootils/src/index.ts

* Add testIgnore for lite

* Fix chatbot_multimodal.spec.ts

* Stop raising an exception when trying to cache examples but just show warning

* Update comment

* Mark the test slow when the page is initialized in it

* Add logs

* Set timeout

* add changeset

* Add the CI file .github/workflows/test-lite.yml

* Add E2E testing for Lite to the test-functional job

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

.changeset/whole-swans-feel.md

@@ -0,0 +1,7 @@
+---
+"@gradio/app": minor
+"@gradio/tootils": minor
+"gradio": minor
+---
+
+feat:E2E tests for Lite

.config/playwright.config.js

@@ -1,6 +1,6 @@
 import { defineConfig } from "@playwright/test";
 
-export default defineConfig({
+const base = defineConfig({
 	use: {
 		screenshot: "only-on-failure",
 		trace: "retain-on-failure",
@@ -19,6 +19,25 @@ export default defineConfig({
 	timeout: 15000,
 	testMatch: /.*.spec.ts/,
 	testDir: "..",
-	globalSetup: "./playwright-setup.js",
 	workers: process.env.CI ? 1 : undefined
 });
+
+const normal = defineConfig(base, {
+	globalSetup: "./playwright-setup.js"
+});
+normal.projects = undefined; // Explicitly unset this field due to https://github.com/microsoft/playwright/issues/28795
+
+const lite = defineConfig(base, {
+	webServer: {
+		command: "pnpm --filter @gradio/app dev:lite",
+		url: "http://localhost:9876/lite.html",
+		reuseExistingServer: !process.env.CI
+	},
+	testIgnore: [
+		"**/clear_components.spec.ts", // `gr.Image()` with remote image is not supported in lite because it calls `httpx.stream` through `processing_utils.save_url_to_cache()`.
+		"**/load_space.spec.ts" // `gr.load()`, which calls `httpx.get` is not supported in lite.
+	]
+});
+lite.projects = undefined; // Explicitly unset this field due to https://github.com/microsoft/playwright/issues/28795
+
+export default !!process.env.GRADIO_E2E_TEST_LITE ? lite : normal;

.github/workflows/test-functional.yml

@@ -3,7 +3,7 @@ name: "test / functional"
 on:
   workflow_run:
     workflows: ["trigger"]
-    types: 
+    types:
       - requested
 
 permissions:
@@ -41,7 +41,7 @@ jobs:
     needs: changes
     if: needs.changes.outputs.should_run == 'true'
     runs-on: ubuntu-latest
-    
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -72,6 +72,7 @@ jobs:
         run: |
           . venv/bin/activate
           pnpm run test:ct
+      - run: pnpm --filter @gradio/app test:browser:lite
       - name: do check
         if: always()
         uses: "gradio-app/github/actions/commit-status@main"

gradio/helpers.py

@@ -291,10 +291,9 @@ class Examples:
                 # And `self.cache()` should be waited for to complete before this method returns,
                 # (otherwise, an error "Cannot cache examples if not in a Blocks context" will be raised anyway)
                 # so `eventloop.create_task(self.cache())` is also not an option.
-                raise wasm_utils.WasmUnsupportedError(
-                    "Caching examples is not supported in the Wasm mode."
-                )
-            client_utils.synchronize_async(self.cache)
+                warnings.warn("Caching examples is not supported in the Wasm mode.")
+            else:
+                client_utils.synchronize_async(self.cache)
 
     async def cache(self) -> None:
         """

js/app/package.json

@@ -18,6 +18,8 @@
 		"test:snapshot": "pnpm exec playwright test snapshots/ --config=../../.config/playwright.config.js",
 		"test:browser": "pnpm exec playwright test test/ --config=../../.config/playwright.config.js",
 		"test:browser:dev": "pnpm exec playwright test test/ --ui --config=../../.config/playwright.config.js",
+		"test:browser:lite": "GRADIO_E2E_TEST_LITE=1 pnpm test:browser",
+		"test:browser:lite:dev": "GRADIO_E2E_TEST_LITE=1 pnpm test:browser:dev",
 		"build:css": "pollen -c pollen.config.cjs -o src/pollen-dev.css"
 	},
 	"dependencies": {

js/app/src/Index.svelte

@@ -62,7 +62,7 @@
 </script>
 
 <script lang="ts">
-	import { onMount, setContext } from "svelte";
+	import { onMount, setContext, createEventDispatcher } from "svelte";
 	import type { api_factory, SpaceStatus } from "@gradio/client";
 	import Embed from "./Embed.svelte";
 	import type { ThemeMode } from "./types";
@@ -74,6 +74,8 @@
 
 	setupi18n();
 
+	const dispatch = createEventDispatcher();
+
 	export let autoscroll: boolean;
 	export let version: string;
 	export let initial_height: string;
@@ -268,6 +270,8 @@
 		css_ready = true;
 		window.__is_colab__ = config.is_colab;
 
+		dispatch("loaded");
+
 		if (config.dev_mode) {
 			setTimeout(() => {
 				const { host } = new URL(api_url);

js/app/src/lite/dev/App.svelte

@@ -79,8 +79,12 @@ def hi(name):
 			controlPageTitle: false,
 			appMode: true
 		});
+		// @ts-ignore
+		window.controller = controller; // For Playwright
 	});
 	onDestroy(() => {
+		// @ts-ignore
+		window.controller = undefined;
 		controller.unmount();
 	});
 

js/app/src/lite/index.ts

@@ -129,7 +129,7 @@ export function create(options: Options): GradioAppController {
 			}
 		});
 	}
-	function launchNewApp(): void {
+	function launchNewApp(): Promise<void> {
 		if (app != null) {
 			app.$destroy();
 		}
@@ -165,6 +165,12 @@ export function create(options: Options): GradioAppController {
 				EventSource_factory
 			}
 		});
+
+		return new Promise((resolve) => {
+			app.$on("loaded", () => {
+				resolve();
+			});
+		});
 	}
 
 	launchNewApp();

js/app/test/chatbot_multimodal.spec.ts

@@ -35,9 +35,13 @@ test("images uploaded by a user should be shown in the chat", async ({
 		.first()
 		.getByRole("paragraph")
 		.textContent();
-	const image_data = await user_message.getAttribute("src");
-	await expect(image_data).toContain("cheetah1.jpg");
-	await expect(bot_message).toBeTruthy();
+	const image_src = await user_message.getAttribute("src");
+	if (process.env.GRADIO_E2E_TEST_LITE) {
+		expect(image_src).toContain(/^blob:.*$/);
+	} else {
+		expect(image_src).toContain("cheetah1.jpg");
+	}
+	expect(bot_message).toBeTruthy();
 });
 
 test("audio uploaded by a user should be shown in the chatbot", async ({

js/app/vite.config.ts

@@ -50,12 +50,14 @@ export default defineConfig(({ mode }) => {
 	const development = mode === "development" || mode === "development:lite";
 	const is_lite = mode.endsWith(":lite");
 
+	const is_e2e_test = process.env.GRADIO_E2E_TEST_LITE;
+
 	return {
 		base: "./",
 
 		server: {
 			port: 9876,
-			open: is_lite ? "/lite.html" : "/"
+			open: is_e2e_test ? false : is_lite ? "/lite.html" : "/"
 		},
 
 		build: {

js/tootils/src/index.ts

@@ -1,7 +1,8 @@
 import { test as base, type Page } from "@playwright/test";
-import { basename } from "path";
 import { spy } from "tinyspy";
-import { readFileSync } from "fs";
+import url from "url";
+import path from "path";
+import fsPromises from "fs/promises";
 
 import type { SvelteComponent } from "svelte";
 import type { SpyFn } from "tinyspy";
@@ -14,12 +15,19 @@ export function wait(n: number): Promise<void> {
 	return new Promise((r) => setTimeout(r, n));
 }
 
-export const test = base.extend<{ setup: void }>({
+const ROOT_DIR = path.resolve(
+	url.fileURLToPath(import.meta.url),
+	"../../../.."
+);
+
+const is_lite = !!process.env.GRADIO_E2E_TEST_LITE;
+
+const test_normal = base.extend<{ setup: void }>({
 	setup: [
 		async ({ page }, use, testInfo): Promise<void> => {
 			const port = process.env.GRADIO_E2E_TEST_PORT;
 			const { file } = testInfo;
-			const test_name = basename(file, ".spec.ts");
+			const test_name = path.basename(file, ".spec.ts");
 
 			await page.goto(`localhost:${port}/${test_name}`);
 
@@ -29,6 +37,102 @@ export const test = base.extend<{ setup: void }>({
 	]
 });
 
+const lite_url = "http://localhost:9876/lite.html";
+// LIte taks a long time to initialize, so we share the page across tests, sacrificing the test isolation.
+let shared_page_for_lite: Page;
+const test_lite = base.extend<{ setup: void }>({
+	page: async ({ browser }, use, testInfo) => {
+		if (shared_page_for_lite == null) {
+			shared_page_for_lite = await browser.newPage();
+		}
+		if (shared_page_for_lite.url() !== lite_url) {
+			await shared_page_for_lite.goto(lite_url);
+			testInfo.setTimeout(600000); // Lite takes a long time to initialize.
+		}
+		await use(shared_page_for_lite);
+	},
+	setup: [
+		async ({ page }, use, testInfo) => {
+			const { file } = testInfo;
+
+			console.debug("Setting up a test in the Lite mode", file);
+			const test_name = path.basename(file, ".spec.ts");
+			const demo_dir = path.resolve(ROOT_DIR, `./demo/${test_name}`);
+			const demo_file_paths = await fsPromises
+				.readdir(demo_dir, { withFileTypes: true, recursive: true })
+				.then((dirents) =>
+					dirents.filter(
+						(dirent) =>
+							dirent.isFile() &&
+							!dirent.name.endsWith(".ipynb") &&
+							!dirent.name.endsWith(".pyc")
+					)
+				)
+				.then((dirents) =>
+					dirents.map((dirent) => path.join(dirent.path, dirent.name))
+				);
+			console.debug("Reading demo files", demo_file_paths);
+			const demo_files = await Promise.all(
+				demo_file_paths.map(async (filepath) => {
+					const relpath = path.relative(demo_dir, filepath);
+					const buffer = await fsPromises.readFile(filepath);
+					return [
+						relpath,
+						buffer.toString("base64") // To pass to the browser, we need to convert the buffer to base64.
+					];
+				})
+			);
+
+			// Mount the demo files and run the app in the mounted Gradio-lite app via its controller.
+			const controllerHandle = await page.waitForFunction(
+				// @ts-ignore
+				() => window.controller // This controller object is set in the dev app.
+			);
+			console.debug("Controller obtained. Setting up the app.");
+			await controllerHandle.evaluate(
+				async (controller: any, files: [string, string][]) => {
+					function base64ToUint8Array(base64: string): Uint8Array {
+						// Ref: https://stackoverflow.com/a/21797381/13103190
+						const binaryString = atob(base64);
+						const bytes = new Uint8Array(binaryString.length);
+						for (var i = 0; i < binaryString.length; i++) {
+							bytes[i] = binaryString.charCodeAt(i);
+						}
+						return bytes;
+					}
+
+					for (const [filepath, data_b64] of files) {
+						const data = base64ToUint8Array(data_b64);
+						if (filepath === "requirements.txt") {
+							const text = new TextDecoder().decode(data);
+							const requirements = text
+								.split("\n")
+								.map((line) => line.trim())
+								.filter((line) => line);
+							console.debug("Installing requirements", requirements);
+							await controller.install(requirements);
+						} else {
+							console.debug("Writing a file", filepath);
+							await controller.write(filepath, data, {});
+						}
+					}
+
+					await controller.run_file("run.py");
+				},
+				demo_files
+			);
+
+			console.debug("App setup done. Starting the test,", test_name);
+			await use();
+
+			controllerHandle.dispose();
+		},
+		{ auto: true }
+	]
+});
+
+export const test = is_lite ? test_lite : test_normal;
+
 export async function wait_for_event(
 	component: SvelteComponent,
 	event: string
@@ -66,7 +170,7 @@ export const drag_and_drop_file = async (
 	fileName: string,
 	fileType = ""
 ): Promise<void> => {
-	const buffer = readFileSync(filePath).toString("base64");
+	const buffer = (await fsPromises.readFile(filePath)).toString("base64");
 
 	const dataTransfer = await page.evaluateHandle(
 		async ({ bufferData, localFileName, localFileType }) => {