1220 mirror webcam (#1686)

* support webcam mirroring for images

* flip video in preprocessor

* finalise webcam mirroring

* address review comments

* fix formatting

* improve video UI

* fix tests

* fix tests again

demo/webcam/run.py

@@ -3,11 +3,15 @@ import numpy as np
 import gradio as gr
 
 
-def snap(image):
-    return np.flipud(image)
+def snap(image, video):
+    return [image, video]
 
 
-demo = gr.Interface(snap, gr.Image(source="webcam", tool=None), "image")
+demo = gr.Interface(
+    snap,
+    [gr.Image(source="webcam", tool=None), gr.Video(source="webcam")],
+    ["image", "video"],
+)
 
 if __name__ == "__main__":
     demo.launch()

gradio/components.py

@@ -10,6 +10,7 @@ import math
 import numbers
 import operator
 import os
+import pathlib
 import shutil
 import tempfile
 import warnings
@@ -1337,6 +1338,7 @@ class Image(Editable, Clearable, Changeable, Streamable, IOComponent):
         visible: bool = True,
         streaming: bool = False,
         elem_id: Optional[str] = None,
+        mirror_webcam: bool = True,
         **kwargs,
     ):
         """
@@ -1354,7 +1356,9 @@ class Image(Editable, Clearable, Changeable, Streamable, IOComponent):
         visible (bool): If False, component will be hidden.
         streaming (bool): If True when used in a `live` interface, will automatically stream webcam feed. Only valid is source is 'webcam'.
         elem_id (Optional[str]): An optional string that is assigned as the id of this component in the HTML DOM. Can be used for targeting CSS styles.
+        mirror_webcam (bool): If True webcam will be mirrored. Default is True.
         """
+        self.mirror_webcam = mirror_webcam
         self.type = type
         self.value = self.postprocess(value)
         self.shape = shape
@@ -1388,6 +1392,7 @@ class Image(Editable, Clearable, Changeable, Streamable, IOComponent):
             "tool": self.tool,
             "value": self.value,
             "streaming": self.streaming,
+            "mirror_webcam": self.mirror_webcam,
             **IOComponent.get_config(self),
         }
 
@@ -1461,6 +1466,8 @@ class Image(Editable, Clearable, Changeable, Streamable, IOComponent):
             im = processing_utils.resize_and_crop(im, self.shape)
         if self.invert_colors:
             im = PIL.ImageOps.invert(im)
+        if self.source == "webcam" and self.mirror_webcam is True:
+            im = PIL.ImageOps.mirror(im)
 
         if not (self.tool == "sketch"):
             return self.format_image(im, fmt)
@@ -1693,6 +1700,7 @@ class Video(Changeable, Clearable, Playable, IOComponent):
         interactive: Optional[bool] = None,
         visible: bool = True,
         elem_id: Optional[str] = None,
+        mirror_webcam: bool = True,
         **kwargs,
     ):
         """
@@ -1705,9 +1713,11 @@ class Video(Changeable, Clearable, Playable, IOComponent):
         interactive (Optional[bool]): if True, will allow users to upload a video; if False, can only be used to display videos. If not provided, this is inferred based on whether the component is used as an input or output.
         visible (bool): If False, component will be hidden.
         elem_id (Optional[str]): An optional string that is assigned as the id of this component in the HTML DOM. Can be used for targeting CSS styles.
+        mirror_webcam (bool): If True webcma will be mirrored. Default is True.
         """
         self.format = format
         self.source = source
+        self.mirror_webcam = mirror_webcam
         self.value = self.postprocess(value)
         IOComponent.__init__(
             self,
@@ -1723,6 +1733,7 @@ class Video(Changeable, Clearable, Playable, IOComponent):
         return {
             "source": self.source,
             "value": self.value,
+            "mirror_webcam": self.mirror_webcam,
             **IOComponent.get_config(self),
         }
 
@@ -1771,11 +1782,21 @@ class Video(Changeable, Clearable, Playable, IOComponent):
             )
         file_name = file.name
         uploaded_format = file_name.split(".")[-1].lower()
+
         if self.format is not None and uploaded_format != self.format:
             output_file_name = file_name[0 : file_name.rindex(".") + 1] + self.format
             ff = FFmpeg(inputs={file_name: None}, outputs={output_file_name: None})
             ff.run()
             return output_file_name
+        elif self.source == "webcam" and self.mirror_webcam is True:
+            path = pathlib.Path(file_name)
+            output_file_name = str(path.with_stem(f"{path.stem}_flip"))
+            ff = FFmpeg(
+                inputs={file_name: None},
+                outputs={output_file_name: ["-vf", "hflip", "-c:a", "copy"]},
+            )
+            ff.run()
+            return output_file_name
         else:
             return file_name
 

gradio/test_data/blocks_configs.py

@@ -46,6 +46,7 @@ XRAY_CONFIG = {
                 "source": "upload",
                 "tool": "editor",
                 "streaming": False,
+                "mirror_webcam": True,
                 "show_label": True,
                 "name": "image",
                 "visible": True,
@@ -86,6 +87,7 @@ XRAY_CONFIG = {
                 "source": "upload",
                 "tool": "editor",
                 "streaming": False,
+                "mirror_webcam": True,
                 "show_label": True,
                 "name": "image",
                 "visible": True,
@@ -232,6 +234,7 @@ XRAY_CONFIG_DIFF_IDS = {
                 "source": "upload",
                 "tool": "editor",
                 "streaming": False,
+                "mirror_webcam": True,
                 "show_label": True,
                 "name": "image",
                 "visible": True,
@@ -277,6 +280,7 @@ XRAY_CONFIG_DIFF_IDS = {
                 "source": "upload",
                 "tool": "editor",
                 "streaming": False,
+                "mirror_webcam": True,
                 "show_label": True,
                 "name": "image",
                 "visible": True,
@@ -433,6 +437,7 @@ XRAY_CONFIG_WITH_MISTAKE = {
                 "image_mode": "RGB",
                 "source": "upload",
                 "streaming": False,
+                "mirror_webcam": True,
                 "tool": "editor",
                 "name": "image",
                 "style": {},
@@ -480,6 +485,7 @@ XRAY_CONFIG_WITH_MISTAKE = {
                 "source": "upload",
                 "tool": "editor",
                 "streaming": False,
+                "mirror_webcam": True,
                 "name": "image",
                 "style": {},
             },

test/test_components.py

@@ -610,6 +610,7 @@ class TestImage(unittest.TestCase):
                 "visible": True,
                 "value": None,
                 "interactive": None,
+                "mirror_webcam": True,
             },
         )
         self.assertIsNone(image_input.preprocess(None))
@@ -1174,6 +1175,7 @@ class TestVideo(unittest.TestCase):
                 "visible": True,
                 "value": None,
                 "interactive": None,
+                "mirror_webcam": True,
             },
         )
         self.assertIsNone(video_input.preprocess(None))

ui/packages/app/src/components/Image/Image.svelte

@@ -17,6 +17,7 @@
 	export let streaming: boolean;
 	export let pending: boolean;
 	export let style: Styles = {};
+	export let mirror_webcam: boolean;
 
 	export let loading_status: LoadingStatus;
 
@@ -58,6 +59,7 @@
 			drop_text={$_("interface.drop_image")}
 			or_text={$_("or")}
 			upload_text={$_("interface.click_to_upload")}
+			{mirror_webcam}
 		/>
 	{/if}
 </Block>

ui/packages/app/src/components/Video/Video.svelte

@@ -18,6 +18,7 @@
 	export let show_label: boolean;
 	export let loading_status: LoadingStatus;
 	export let style: Styles = {};
+	export let mirror_webcam: boolean;
 
 	export let mode: "static" | "dynamic";
 
@@ -52,6 +53,7 @@
 			drop_text={$_("interface.drop_video")}
 			or_text={$_("or")}
 			upload_text={$_("interface.click_to_upload")}
+			{mirror_webcam}
 			on:change
 			on:clear
 			on:play

ui/packages/icons/src/Maximise.svelte

@@ -0,0 +1,14 @@
+<svg
+	xmlns="http://www.w3.org/2000/svg"
+	width="100%"
+	height="100%"
+	viewBox="0 0 24 24"
+	fill="none"
+	stroke="currentColor"
+	stroke-width="1.5"
+	stroke-linecap="round"
+	stroke-linejoin="round"
+	><path
+		d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3m0 18h3a2 2 0 0 0 2-2v-3M3 16v3a2 2 0 0 0 2 2h3"
+	/></svg
+>

ui/packages/icons/src/Pause.svelte

@@ -0,0 +1,17 @@
+<svg
+	xmlns="http://www.w3.org/2000/svg"
+	width="100%"
+	height="100%"
+	viewBox="0 0 24 24"
+	fill="none"
+	stroke="currentColor"
+	stroke-width="1.5"
+	stroke-linecap="round"
+	stroke-linejoin="round"
+	><rect x="6" y="4" width="4" height="16" /><rect
+		x="14"
+		y="4"
+		width="4"
+		height="16"
+	/></svg
+>

ui/packages/icons/src/Play.svelte

@@ -0,0 +1,11 @@
+<svg
+	xmlns="http://www.w3.org/2000/svg"
+	width="100%"
+	height="100%"
+	viewBox="0 0 24 24"
+	fill="none"
+	stroke="currentColor"
+	stroke-width="1.5"
+	stroke-linecap="round"
+	stroke-linejoin="round"><polygon points="5 3 19 12 5 21 5 3" /></svg
+>

ui/packages/icons/src/index.ts

@@ -17,3 +17,6 @@ export { default as JSON } from "./JSON.svelte";
 export { default as Tree } from "./Tree.svelte";
 export { default as Chat } from "./Chat.svelte";
 export { default as Plot } from "./Plot.svelte";
+export { default as Play } from "./Play.svelte";
+export { default as Pause } from "./Pause.svelte";
+export { default as Maximise } from "./Maximise.svelte";

ui/packages/image/src/Image.svelte

@@ -25,6 +25,7 @@
 	export let upload_text: string = "click to upload";
 	export let streaming: boolean = false;
 	export let pending: boolean = false;
+	export let mirror_webcam: boolean;
 
 	let sketch: Sketch;
 
@@ -133,6 +134,7 @@
 				on:stream={handle_save}
 				{streaming}
 				{pending}
+				{mirror_webcam}
 			/>
 		{/if}
 	{:else if tool === "select"}
@@ -145,13 +147,19 @@
 			editable
 		/>
 
-		<img class="w-full h-full object-contain" src={value} alt="" />
+		<img
+			class="w-full h-full object-contain"
+			src={value}
+			alt=""
+			class:scale-x-[-1]={source === "webcam" && mirror_webcam}
+		/>
 	{:else if tool === "sketch" && value !== null}
 		<img
 			class="absolute w-full h-full object-contain"
 			src={value.image}
 			alt=""
 			on:load={handle_image_load}
+			class:scale-x-[-1]={source === "webcam" && mirror_webcam}
 		/>
 		{#if img_width > 0}
 			<Sketch
@@ -170,6 +178,11 @@
 			/>
 		{/if}
 	{:else}
-		<img class="w-full h-full object-contain" src={value} alt="" />
+		<img
+			class="w-full h-full object-contain"
+			src={value}
+			alt=""
+			class:scale-x-[-1]={source === "webcam" && mirror_webcam}
+		/>
 	{/if}
 </div>

ui/packages/image/src/Webcam.svelte

@@ -8,6 +8,7 @@
 	export let pending: boolean = false;
 
 	export let mode: "image" | "video" = "image";
+	export let mirror_webcam: boolean;
 
 	const dispatch = createEventDispatcher();
 
@@ -102,7 +103,11 @@
 
 <div class="h-full min-h-[15rem] w-full relative">
 	<!-- svelte-ignore a11y-media-has-caption -->
-	<video bind:this={video_source} class="h-full w-full " />
+	<video
+		bind:this={video_source}
+		class="h-full w-full "
+		class:scale-x-[-1]={mirror_webcam}
+	/>
 	{#if !streaming}
 		<button
 			on:click={mode === "image" ? take_picture : take_recording}

ui/packages/video/src/Player.svelte

@@ -0,0 +1,159 @@
+<script lang="ts">
+	import { tick } from "svelte";
+	import { Play, Pause, Maximise, Undo } from "@gradio/icons";
+
+	export let src: string;
+	export let mirror: boolean;
+
+	let time: number = 0;
+	let duration: number;
+	let paused: boolean = true;
+	let video: HTMLVideoElement;
+
+	let show_controls = true;
+	let show_controls_timeout: NodeJS.Timeout;
+
+	function video_move() {
+		clearTimeout(show_controls_timeout);
+		show_controls_timeout = setTimeout(() => (show_controls = false), 2500);
+		show_controls = true;
+	}
+
+	function handleMove(e: TouchEvent | MouseEvent) {
+		if (!duration) return;
+
+		if (e.type === "click") {
+			handle_click(e as MouseEvent);
+			return;
+		}
+
+		if (e.type !== "touchmove" && !((e as MouseEvent).buttons & 1)) return;
+
+		const clientX =
+			e.type === "touchmove"
+				? (e as TouchEvent).touches[0].clientX
+				: (e as MouseEvent).clientX;
+		const { left, right } = (
+			e.currentTarget as HTMLProgressElement
+		).getBoundingClientRect();
+		time = (duration * (clientX - left)) / (right - left);
+	}
+
+	function play_pause() {
+		if (paused) video.play();
+		else video.pause();
+	}
+
+	function handle_click(e: MouseEvent) {
+		const { left, right } = (
+			e.currentTarget as HTMLProgressElement
+		).getBoundingClientRect();
+		time = (duration * (e.clientX - left)) / (right - left);
+	}
+
+	function format(seconds: number) {
+		if (isNaN(seconds) || !isFinite(seconds)) return "...";
+
+		const minutes = Math.floor(seconds / 60);
+		let _seconds: number | string = Math.floor(seconds % 60);
+		if (seconds < 10) _seconds = `0${_seconds}`;
+
+		return `${minutes}:${_seconds}`;
+	}
+
+	async function _load() {
+		await tick();
+		video.currentTime = 9999;
+
+		setTimeout(async () => {
+			video.currentTime = 0.0;
+		}, 50);
+	}
+
+	$: src && _load();
+</script>
+
+<div>
+	<video
+		{src}
+		preload="auto"
+		on:mousemove={video_move}
+		on:click={play_pause}
+		on:play
+		on:pause
+		on:ended
+		bind:currentTime={time}
+		bind:duration
+		bind:paused
+		bind:this={video}
+		class="w-full h-full object-contain bg-black"
+		class:mirror
+	>
+		<track kind="captions" />
+	</video>
+
+	<div
+		class="wrap absolute bottom-0 transition duration-500  m-1.5 bg-slate-800 px-1 py-2.5 rounded-md"
+		style="opacity: {duration && show_controls ? 1 : 0}"
+		on:mousemove={video_move}
+	>
+		<div class="flex w-full justify-space h-full items-center px-1.5 ">
+			<span
+				class=" w-6 cursor-pointer text-white flex justify-center"
+				on:click={play_pause}
+			>
+				{#if time === duration}
+					<Undo />
+				{:else if paused}
+					<Play />
+				{:else}
+					<Pause />
+				{/if}
+			</span>
+
+			<span class="font-mono shrink-0 text-xs mx-3 text-white"
+				>{format(time)} / {format(duration)}</span
+			>
+			<progress
+				value={time / duration || 0}
+				on:mousemove={handleMove}
+				on:touchmove|preventDefault={handleMove}
+				on:click|stopPropagation|preventDefault={handle_click}
+				class="rounded h-2 w-full mx-3"
+			/>
+
+			<div
+				class="w-6 cursor-pointer text-white"
+				on:click={() => video.requestFullscreen()}
+			>
+				<Maximise />
+			</div>
+		</div>
+	</div>
+</div>
+
+<style lang="postcss">
+	span {
+		text-shadow: 0 0 8px rgba(0, 0, 0, 0.5);
+	}
+
+	progress::-webkit-progress-bar {
+		border-radius: 2px;
+		background-color: rgba(255, 255, 255, 0.2);
+		overflow: hidden;
+	}
+
+	progress::-webkit-progress-value {
+		/* border-radius: 2px; */
+
+		background-color: rgba(255, 255, 255, 0.9);
+	}
+
+	.mirror {
+		transform: scaleX(-1);
+	}
+
+	.wrap {
+		width: calc(100% - 0.375rem * 2);
+	}
+</style>

ui/packages/video/src/StaticVideo.svelte

@@ -4,6 +4,8 @@
 	import type { FileData } from "@gradio/upload";
 	import { Video } from "@gradio/icons";
 
+	import Player from "./Player.svelte";
+
 	export let value: FileData | null = null;
 	export let label: string | undefined = undefined;
 	export let show_label: boolean;
@@ -25,14 +27,5 @@
 	</div>
 {:else}
 	<!-- svelte-ignore a11y-media-has-caption -->
-	<video
-		class="w-full h-full object-contain bg-black"
-		controls
-		playsInline
-		preload="auto"
-		src={value.data}
-		on:play
-		on:pause
-		on:ended
-	/>
+	<Player src={value.data} on:play on:pause on:ended mirror={false} />
 {/if}

ui/packages/video/src/Video.svelte

@@ -7,11 +7,13 @@
 	import { Video } from "@gradio/icons";
 
 	import { prettyBytes, playable } from "./utils";
+	import Player from "./Player.svelte";
 
 	export let value: FileData | null = null;
 	export let source: string;
 	export let label: string | undefined = undefined;
 	export let show_label: boolean;
+	export let mirror_webcam: boolean;
 
 	export let drop_text: string = "Drop a video file";
 	export let or_text: string = "or";
@@ -57,6 +59,7 @@
 		</Upload>
 	{:else if source === "webcam"}
 		<Webcam
+			{mirror_webcam}
 			mode="video"
 			on:capture={({ detail }) => dispatch("change", detail)}
 		/>
@@ -65,16 +68,7 @@
 	<ModifyUpload on:clear={handle_clear} />
 	{#if playable(value.name)}
 		<!-- svelte-ignore a11y-media-has-caption -->
-		<video
-			class="w-full h-full object-contain bg-black"
-			controls
-			playsInline
-			preload="auto"
-			src={value.data}
-			on:play
-			on:pause
-			on:ended
-		/>
+		<Player src={value.data} on:play on:pause on:ended mirror={mirror_webcam} />
 	{:else if value.size}
 		<div class="file-name text-4xl p-6 break-all">{value.name}</div>
 		<div class="file-size text-2xl p-2">