Allow start/pause of streaming image input. Only access the webcam while it's needed (#7228)

.changeset/major-rats-like.md

@@ -0,0 +1,6 @@
+---
+"@gradio/image": minor
+"gradio": minor
+---
+
+feat:Allow start/pause of streaming image input. Only access the webcam while it's needed

gradio/interface.py

@@ -649,9 +649,11 @@ class Interface(Blocks):
                 )
             else:
                 events: list[Callable] = []
+                streaming_event = False
                 for component in self.input_components:
                     if component.has_event("stream") and component.streaming:  # type: ignore
                         events.append(component.stream)  # type: ignore
+                        streaming_event = True
                     elif component.has_event("change"):
                         events.append(component.change)  # type: ignore
                 on(
@@ -662,6 +664,7 @@ class Interface(Blocks):
                     api_name=self.api_name,
                     preprocess=not (self.api_mode),
                     postprocess=not (self.api_mode),
+                    show_progress="hidden" if streaming_event else "full",
                 )
         else:
             if _submit_btn is None:

js/image/Image.stories.svelte

@@ -72,8 +72,7 @@
 		const webcamButton = await canvas.findByLabelText("Capture from camera");
 		userEvent.click(webcamButton);
 
-		userEvent.click(await canvas.findByTitle("select video source"));
-		userEvent.click(await canvas.findByLabelText("select source"));
+		userEvent.click(await canvas.findByTitle("grant webcam access"));
 		userEvent.click(await canvas.findByLabelText("Upload file"));
 		userEvent.click(await canvas.findByLabelText("Paste from clipboard"));
 	}}

js/image/shared/Webcam.svelte

@@ -1,9 +1,17 @@
 <script lang="ts">
 	import { createEventDispatcher, onMount } from "svelte";
-	import { Camera, Circle, Square, DropdownArrow } from "@gradio/icons";
+	import {
+		Camera,
+		Circle,
+		Square,
+		DropdownArrow,
+		Webcam as WebcamIcon
+	} from "@gradio/icons";
 	import type { I18nFormatter } from "@gradio/utils";
 	import type { FileData } from "@gradio/client";
 	import { prepare_files, upload } from "@gradio/client";
+	import WebcamPermissions from "./WebcamPermissions.svelte";
+	import { fade } from "svelte/transition";
 
 	let video_source: HTMLVideoElement;
 	let canvas: HTMLCanvasElement;
@@ -42,6 +50,7 @@
 			video_source.srcObject = stream;
 			video_source.muted = true;
 			video_source.play();
+			webcam_accessed = true;
 		} catch (err) {
 			if (err instanceof DOMException && err.name == "NotAllowedError") {
 				dispatch("error", i18n("image.allow_webcam_access"));
@@ -53,8 +62,11 @@
 
 	function take_picture(): void {
 		var context = canvas.getContext("2d")!;
-
-		if (video_source.videoWidth && video_source.videoHeight) {
+		if (
+			(!streaming || (streaming && recording)) &&
+			video_source.videoWidth &&
+			video_source.videoHeight
+		) {
 			canvas.width = video_source.videoWidth;
 			canvas.height = video_source.videoHeight;
 			context.drawImage(
@@ -131,7 +143,23 @@
 		recording = !recording;
 	}
 
-	access_webcam();
+	let webcam_accessed = false;
+
+	function record_video_or_photo(): void {
+		if (mode === "image" && streaming) {
+			recording = !recording;
+		}
+		if (mode === "image") {
+			take_picture();
+		} else {
+			take_recording();
+		}
+		if (!recording && stream) {
+			stream.getTracks().forEach((track) => track.stop());
+			video_source.srcObject = null;
+			webcam_accessed = false;
+		}
+	}
 
 	if (streaming && mode === "image") {
 		window.setInterval(() => {
@@ -185,14 +213,22 @@
 <div class="wrap">
 	<!-- svelte-ignore a11y-media-has-caption -->
 	<!-- need to suppress for video streaming https://github.com/sveltejs/svelte/issues/5967 -->
-	<video bind:this={video_source} class:flip={mirror_webcam} />
-	{#if !streaming}
+	<video
+		bind:this={video_source}
+		class:flip={mirror_webcam}
+		class:hide={!webcam_accessed}
+	/>
+	{#if !webcam_accessed}
+		<div in:fade={{ delay: 100, duration: 200 }} title="grant webcam access">
+			<WebcamPermissions on:click={async () => access_webcam()} />
+		</div>
+	{:else}
 		<div class="button-wrap">
 			<button
-				on:click={mode === "image" ? take_picture : take_recording}
+				on:click={record_video_or_photo}
 				aria-label={mode === "image" ? "capture photo" : "start recording"}
 			>
-				{#if mode === "video"}
+				{#if mode === "video" || streaming}
 					{#if recording}
 						<div class="icon red" title="stop recording">
 							<Square />
@@ -208,7 +244,6 @@
 					</div>
 				{/if}
 			</button>
-
 			{#if !recording}
 				<button
 					on:click={select_source}
@@ -253,6 +288,10 @@
 		height: var(--size-full);
 	}
 
+	.hide {
+		display: none;
+	}
+
 	video {
 		width: var(--size-full);
 		height: var(--size-full);
@@ -353,4 +392,10 @@
 		height: var(--size-5);
 		opacity: 0.8;
 	}
+
+	@media (--screen-md) {
+		.wrap {
+			font-size: var(--text-lg);
+		}
+	}
 </style>

js/image/shared/WebcamPermissions.svelte

@@ -0,0 +1,46 @@
+<script lang="ts">
+	import { Webcam } from "@gradio/icons";
+	import { createEventDispatcher } from "svelte";
+
+	const dispatch = createEventDispatcher<{
+		click: undefined;
+	}>();
+</script>
+
+<button style:height="100%" on:click={() => dispatch("click")}>
+	<div class="wrap">
+		<span class="icon-wrap">
+			<Webcam />
+		</span>
+		{"Click to Access Webcam"}
+	</div>
+</button>
+
+<style>
+	button {
+		cursor: pointer;
+		width: var(--size-full);
+	}
+
+	.wrap {
+		display: flex;
+		flex-direction: column;
+		justify-content: center;
+		align-items: center;
+		min-height: var(--size-60);
+		color: var(--block-label-text-color);
+		height: 100%;
+		padding-top: var(--size-3);
+	}
+
+	.icon-wrap {
+		width: 30px;
+		margin-bottom: var(--spacing-lg);
+	}
+
+	@media (--screen-md) {
+		.wrap {
+			font-size: var(--text-lg);
+		}
+	}
+</style>