Video/Audio fixes (#6234)

* Add code

* Add code

* add changeset

* Add code

* Add code

* prevent resetting source when clearing value

* Add code

* Add drag-and-drop tests

* add changeset

* remove console log

* Format

* Add code

* add changeset

* Audio components

* add changeset

* add changeset

* Add return type

* Add code

* promise

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Co-authored-by: Hannah <hannahblair@users.noreply.github.com>

.changeset/polite-radios-kiss.md

@@ -0,0 +1,10 @@
+---
+"@gradio/app": patch
+"@gradio/audio": patch
+"@gradio/tootils": patch
+"@gradio/upload": patch
+"@gradio/video": patch
+"gradio": patch
+---
+
+fix:Video/Audio fixes

demo/audio_component_events/run.ipynb

@@ -0,0 +1 @@
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: audio_component_events"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", "    with gr.Row():\n", "        with gr.Column():\n", "            input_video = gr.Audio(label=\"Input Audio\", sources=[\"upload\", \"microphone\"])\n", "        with gr.Column():\n", "            output_video = gr.Audio(label=\"Output Audio\", sources=[\"upload\", \"microphone\"])\n", "        with gr.Column():\n", "            num_change = gr.Number(label=\"# Change Events\", value=0)\n", "            num_load = gr.Number(label=\"# Upload Events\", value=0)\n", "            num_play = gr.Number(label=\"# Play Events\", value=0)\n", "            num_pause = gr.Number(label=\"# Pause Events\", value=0)\n", "        input_video.upload(lambda s, n: (s, n + 1), [input_video, num_load], [output_video, num_load])\n", "        input_video.change(lambda n: n + 1, num_change, num_change)\n", "        input_video.play(lambda n: n + 1, num_play, num_play)\n", "        input_video.pause(lambda n: n + 1, num_pause, num_pause)\n", "        input_video.change(lambda n: n + 1, num_change, num_change)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/audio_component_events/run.py

@@ -0,0 +1,21 @@
+import gradio as gr
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        with gr.Column():
+            input_video = gr.Audio(label="Input Audio", sources=["upload", "microphone"])
+        with gr.Column():
+            output_video = gr.Audio(label="Output Audio", sources=["upload", "microphone"])
+        with gr.Column():
+            num_change = gr.Number(label="# Change Events", value=0)
+            num_load = gr.Number(label="# Upload Events", value=0)
+            num_play = gr.Number(label="# Play Events", value=0)
+            num_pause = gr.Number(label="# Pause Events", value=0)
+        input_video.upload(lambda s, n: (s, n + 1), [input_video, num_load], [output_video, num_load])
+        input_video.change(lambda n: n + 1, num_change, num_change)
+        input_video.play(lambda n: n + 1, num_play, num_play)
+        input_video.pause(lambda n: n + 1, num_pause, num_pause)
+        input_video.change(lambda n: n + 1, num_change, num_change)
+
+if __name__ == "__main__":
+    demo.launch()

demo/video_component_events/run.ipynb

@@ -0,0 +1 @@
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: video_component_events"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", "    with gr.Row():\n", "        with gr.Column():\n", "            input_video = gr.Video(label=\"Input Video\")\n", "        with gr.Column():\n", "            output_video = gr.Video(label=\"Output Video\")\n", "        with gr.Column():\n", "            num_change = gr.Number(label=\"# Change Events\", value=0)\n", "            num_load = gr.Number(label=\"# Upload Events\", value=0)\n", "            num_play = gr.Number(label=\"# Play Events\", value=0)\n", "            num_pause = gr.Number(label=\"# Pause Events\", value=0)\n", "        input_video.upload(lambda s, n: (s, n + 1), [input_video, num_load], [output_video, num_load])\n", "        input_video.change(lambda n: n + 1, num_change, num_change)\n", "        input_video.play(lambda n: n + 1, num_play, num_play)\n", "        input_video.pause(lambda n: n + 1, num_pause, num_pause)\n", "        input_video.change(lambda n: n + 1, num_change, num_change)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/video_component_events/run.py

@@ -0,0 +1,21 @@
+import gradio as gr
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        with gr.Column():
+            input_video = gr.Video(label="Input Video")
+        with gr.Column():
+            output_video = gr.Video(label="Output Video")
+        with gr.Column():
+            num_change = gr.Number(label="# Change Events", value=0)
+            num_load = gr.Number(label="# Upload Events", value=0)
+            num_play = gr.Number(label="# Play Events", value=0)
+            num_pause = gr.Number(label="# Pause Events", value=0)
+        input_video.upload(lambda s, n: (s, n + 1), [input_video, num_load], [output_video, num_load])
+        input_video.change(lambda n: n + 1, num_change, num_change)
+        input_video.play(lambda n: n + 1, num_play, num_play)
+        input_video.pause(lambda n: n + 1, num_pause, num_pause)
+        input_video.change(lambda n: n + 1, num_change, num_change)
+
+if __name__ == "__main__":
+    demo.launch()

gradio/processing_utils.py

@@ -258,6 +258,7 @@ def move_resource_to_block_cache(url_or_file_path: str | Path, block: Component)
         temp_file_path = save_url_to_cache(
             url_or_file_path, cache_dir=block.GRADIO_CACHE
         )
+
         block.temp_files.add(temp_file_path)
     else:
         url_or_file_path = str(abspath(url_or_file_path))
@@ -265,9 +266,9 @@ def move_resource_to_block_cache(url_or_file_path: str | Path, block: Component)
             temp_file_path = save_file_to_cache(
                 url_or_file_path, cache_dir=block.GRADIO_CACHE
             )
-            block.temp_files.add(temp_file_path)
         else:
             temp_file_path = url_or_file_path
+        block.temp_files.add(temp_file_path)
 
     return temp_file_path
 

js/app/src/Blocks.svelte

@@ -613,8 +613,8 @@
 			if (event === "share") {
 				const { title, description } = data as ShareData;
 				trigger_share(title, description);
-			} else if (event === "error") {
-				messages = [new_message(data, -1, "error"), ...messages];
+			} else if (event === "error" || event === "warning") {
+				messages = [new_message(data, -1, event), ...messages];
 			} else {
 				const deps = target_map[id]?.[event];
 				deps?.forEach((dep_id) => {

js/app/test/audio_component_events.spec.ts

@@ -0,0 +1,59 @@
+import { test, expect, drag_and_drop_file } from "@gradio/tootils";
+
+test("Audio click-to-upload uploads audio successfuly.", async ({ page }) => {
+	await page
+		.getByRole("button", { name: "Drop Audio Here - or - Click to Upload" })
+		.click();
+	const uploader = await page.locator("input[type=file]");
+	await Promise.all([
+		uploader.setInputFiles(["../../test/test_files/audio_sample.wav"]),
+		page.waitForResponse("**/upload")
+	]);
+
+	await expect(page.getByLabel("# Change Events")).toHaveValue("1");
+	await expect(page.getByLabel("# Upload Events")).toHaveValue("1");
+
+	await page.getByLabel("Clear").click();
+	await expect(page.getByLabel("# Change Events")).toHaveValue("2");
+	await page
+		.getByRole("button", { name: "Drop Audio Here - or - Click to Upload" })
+		.click();
+
+	await Promise.all([
+		uploader.setInputFiles(["../../test/test_files/audio_sample.wav"]),
+		page.waitForResponse("**/upload")
+	]);
+
+	await expect(page.getByLabel("# Change Events")).toHaveValue("3");
+	await expect(page.getByLabel("# Upload Events")).toHaveValue("2");
+});
+
+test("Audio drag-and-drop uploads a file to the server correctly.", async ({
+	page
+}) => {
+	await Promise.all([
+		drag_and_drop_file(
+			page,
+			"input[type=file]",
+			"../../test/test_files/audio_sample.wav",
+			"audio_sample.wav",
+			"audio/wav"
+		),
+		page.waitForResponse("**/upload")
+	]);
+	await expect(page.getByLabel("# Change Events")).toHaveValue("1");
+	await expect(page.getByLabel("# Upload Events")).toHaveValue("1");
+});
+
+test("Audio drag-and-drop displays a warning when the file is of the wrong mime type.", async ({
+	page
+}) => {
+	await drag_and_drop_file(
+		page,
+		"input[type=file]",
+		"../../test/test_files/audio_sample.wav",
+		"audio_sample.wav"
+	);
+	const toast = page.getByTestId("toast-body");
+	expect(toast).toContainText("warning");
+});

js/app/test/video_component_events.spec.ts

@@ -0,0 +1,69 @@
+import { test, expect, drag_and_drop_file } from "@gradio/tootils";
+
+test("Video click-to-upload uploads video successfuly. Clear, play, and pause buttons dispatch events correctly.", async ({
+	page
+}) => {
+	await page
+		.getByRole("button", { name: "Drop Video Here - or - Click to Upload" })
+		.click();
+	const uploader = await page.locator("input[type=file]");
+	await Promise.all([
+		uploader.setInputFiles(["./test/files/file_test.ogg"]),
+		page.waitForResponse("**/upload")
+	]);
+
+	await expect(page.getByLabel("# Change Events")).toHaveValue("1");
+	await expect(page.getByLabel("# Upload Events")).toHaveValue("1");
+
+	await page.getByLabel("play-pause-replay-button").nth(0).click();
+	await page.getByLabel("play-pause-replay-button").nth(0).click();
+	await expect(page.getByLabel("# Play Events")).toHaveValue("1");
+	await expect(page.getByLabel("# Pause Events")).toHaveValue("1");
+
+	await page.getByLabel("Clear").click();
+	await expect(page.getByLabel("# Change Events")).toHaveValue("2");
+	await page
+		.getByRole("button", { name: "Drop Video Here - or - Click to Upload" })
+		.click();
+
+	await Promise.all([
+		uploader.setInputFiles(["./test/files/file_test.ogg"]),
+		page.waitForResponse("**/upload")
+	]);
+
+	await expect(page.getByLabel("# Change Events")).toHaveValue("3");
+	await expect(page.getByLabel("# Upload Events")).toHaveValue("2");
+
+	await page.getByLabel("play-pause-replay-button").first().click();
+	await page.getByLabel("play-pause-replay-button").first().click();
+	await expect(page.getByLabel("# Play Events")).toHaveValue("2");
+	await expect(page.getByLabel("# Pause Events")).toHaveValue("2");
+});
+
+test("Video drag-and-drop uploads a file to the server correctly.", async ({
+	page
+}) => {
+	await drag_and_drop_file(
+		page,
+		"input[type=file]",
+		"./test/files/file_test.ogg",
+		"file_test.ogg",
+		"video/*"
+	);
+	await page.waitForResponse("**/upload");
+	await expect(page.getByLabel("# Change Events")).toHaveValue("1");
+	await expect(page.getByLabel("# Upload Events")).toHaveValue("1");
+});
+
+test("Video drag-and-drop displays a warning when the file is of the wrong mime type.", async ({
+	page
+}) => {
+	await drag_and_drop_file(
+		page,
+		"input[type=file]",
+		"./test/files/file_test.ogg",
+		"file_test.ogg"
+	);
+	const toast = page.getByTestId("toast-body");
+	expect(toast).toContainText("warning");
+});

js/audio/Index.svelte

@@ -41,6 +41,7 @@
 		change: typeof value;
 		stream: typeof value;
 		error: string;
+		warning: string;
 		edit: never;
 		play: never;
 		pause: never;
@@ -100,6 +101,16 @@
 		dragToSeek: true,
 		mediaControls: waveform_options.show_controls
 	};
+
+	function handle_error({ detail }: CustomEvent<string>): void {
+		const [level, status] = detail.includes("Invalid file type")
+			? ["warning", "complete"]
+			: ["error", "error"];
+		loading_status = loading_status || {};
+		loading_status.status = status as LoadingStatus["status"];
+		loading_status.message = detail;
+		gradio.dispatch(level as "error" | "warning", detail);
+	}
 </script>
 
 {#if !interactive}
@@ -178,11 +189,7 @@
 			on:stop_recording={() => gradio.dispatch("stop_recording")}
 			on:upload={() => gradio.dispatch("upload")}
 			on:clear={() => gradio.dispatch("clear")}
-			on:error={({ detail }) => {
-				loading_status = loading_status || {};
-				loading_status.status = "error";
-				gradio.dispatch("error", detail);
-			}}
+			on:error={handle_error}
 			i18n={gradio.i18n}
 			{waveform_settings}
 		>

js/audio/interactive/InteractiveAudio.svelte

@@ -220,12 +220,12 @@
 			/>
 		{/if}
 	{:else if active_source === "upload"}
-		<ModifyUpload {i18n} on:clear={clear} absolute={true} />
 		<!-- explicitly listed out audio mimetypes due to iOS bug not recognizing audio/* -->
 		<Upload
 			filetype="audio/aac,audio/midi,audio/mpeg,audio/ogg,audio/wav,audio/x-wav,audio/opus,audio/webm,audio/flac,audio/vnd.rn-realaudio,audio/x-ms-wma,audio/x-aiff,audio/amr,audio/*"
 			on:load={handle_load}
 			bind:dragging
+			on:error={({ detail }) => dispatch("error", detail)}
 			{root}
 		>
 			<slot />

js/tootils/src/index.ts

@@ -1,6 +1,7 @@
-import { test as base } from "@playwright/test";
+import { test as base, type Page } from "@playwright/test";
 import { basename } from "path";
 import { spy } from "tinyspy";
+import { readFileSync } from "fs";
 
 import type { SvelteComponent } from "svelte";
 import type { SpyFn } from "tinyspy";
@@ -57,3 +58,32 @@ export interface ActionReturn<
 
 export { expect } from "@playwright/test";
 export * from "./render";
+
+export const drag_and_drop_file = async (
+	page: Page,
+	selector: string,
+	filePath: string,
+	fileName: string,
+	fileType = ""
+): Promise<void> => {
+	const buffer = readFileSync(filePath).toString("base64");
+
+	const dataTransfer = await page.evaluateHandle(
+		async ({ bufferData, localFileName, localFileType }) => {
+			const dt = new DataTransfer();
+
+			const blobData = await fetch(bufferData).then((res) => res.blob());
+
+			const file = new File([blobData], localFileName, { type: localFileType });
+			dt.items.add(file);
+			return dt;
+		},
+		{
+			bufferData: `data:application/octet-stream;base64,${buffer}`,
+			localFileName: fileName,
+			localFileType: fileType
+		}
+	);
+
+	await page.dispatchEvent(selector, "drop", { dataTransfer });
+};

js/upload/src/Upload.svelte

@@ -46,9 +46,7 @@
 		if (!files.length) {
 			return;
 		}
-
 		let _files: File[] = files.map((f) => new File([f], f.name));
-
 		let file_data = await prepare_files(_files);
 		return await handle_upload(file_data);
 	}
@@ -80,7 +78,7 @@
 		if (!e.dataTransfer?.files) return;
 
 		const files_to_load = Array.from(e.dataTransfer.files).filter((f) => {
-			if (is_valid_mimetype(filetype, f.type)) {
+			if (filetype?.split(",").some((m) => is_valid_mimetype(m, f.type))) {
 				return true;
 			}
 			dispatch("error", `Invalid file type only ${filetype} allowed.`);

js/video/Index.svelte

@@ -47,6 +47,7 @@
 		stop_recording: never;
 		share: ShareData;
 		error: string;
+		warning: string;
 	}>;
 	export let interactive: boolean;
 	export let mirror_webcam: boolean;
@@ -109,6 +110,16 @@
 			value = null;
 		}
 	}
+
+	function handle_error({ detail }: CustomEvent<string>): void {
+		const [level, status] = detail.includes("Invalid file type")
+			? ["warning", "complete"]
+			: ["error", "error"];
+		loading_status = loading_status || {};
+		loading_status.status = status as LoadingStatus["status"];
+		loading_status.message = detail;
+		gradio.dispatch(level as "error" | "warning", detail);
+	}
 </script>
 
 {#if !interactive}
@@ -175,11 +186,7 @@
 			subtitle={_subtitle}
 			on:change={handle_change}
 			on:drag={({ detail }) => (dragging = detail)}
-			on:error={({ detail }) => {
-				loading_status = loading_status || {};
-				loading_status.status = "error";
-				loading_status.message = detail;
-			}}
+			on:error={handle_error}
 			{label}
 			{show_label}
 			{sources}

js/video/shared/InteractiveVideo.svelte

@@ -48,7 +48,6 @@
 
 	function handle_clear(): void {
 		value = null;
-		active_source = sources[0];
 		dispatch("change", null);
 		dispatch("clear");
 	}
@@ -68,6 +67,7 @@
 			bind:dragging
 			filetype="video/x-m4v,video/*"
 			on:load={handle_load}
+			on:error={({ detail }) => dispatch("error", detail)}
 			{root}
 		>
 			<slot />
@@ -124,7 +124,7 @@
 		>
 		<button
 			class="icon"
-			aria-label="Record audio"
+			aria-label="Record video"
 			on:click={() => {
 				handle_clear();
 				active_source = "webcam";