Create Streamables (#1279)

* changes

* fix

* fix for vars too

* changes

* fix tests

Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

demo/calculator/run.py

@@ -1,6 +1,5 @@
 import gradio as gr
 
-
 def calculator(num1, operation, num2):
     if operation == "add":
         return num1 + num2

demo/live_with_vars/run.py

@@ -0,0 +1,6 @@
+import gradio as gr
+
+gr.Interface(
+    lambda x, y: (x + y if y is not None else x, x + y if y is not None else x), 
+    ["textbox", "state"], 
+    ["textbox", "state"], live=True).launch()

demo/stream_audio/run.py

@@ -0,0 +1,20 @@
+import gradio as gr
+import numpy as np
+
+with gr.Blocks() as demo:
+    inp = gr.Audio(source="microphone")
+    out = gr.Audio()
+    stream = gr.Variable()
+
+    def add_to_stream(audio, instream):
+        if audio is None:
+            return gr.update(), instream
+        if instream is None:
+            ret = audio
+        else:
+            ret = (audio[0], np.concatenate((instream[1], audio[1])))
+        return ret, ret
+    inp.stream(add_to_stream, [inp, stream], [out, stream])
+
+if __name__ == "__main__":
+    demo.launch()

demo/stream_frames/run.py

@@ -0,0 +1,13 @@
+import gradio as gr
+import numpy as np
+
+with gr.Blocks() as demo:
+    inp = gr.Image(source="webcam")
+    out = gr.Image()
+
+    def flip(im):
+        return np.flipud(im)
+    inp.stream(flip, [inp], [out])
+
+if __name__ == "__main__":
+    demo.launch()

demo/streaming_stt/run.py

@@ -1,7 +1,7 @@
 from deepspeech import Model
 import gradio as gr
 import numpy as np
-import urllib.request 
+import urllib.request
 
 model_file_path = "deepspeech-0.9.3-models.pbmm"
 lm_file_path = "deepspeech-0.9.3-models.scorer"
@@ -45,7 +45,13 @@ def transcribe(speech, stream):
     text = stream.intermediateDecode()
     return text, stream
 
-demo = gr.Interface(transcribe, ["microphone", "state"], ["text", "state"], live=True)
+
+demo = gr.Interface(
+    transcribe,
+    [gr.Audio(source="microphone", streaming=True), "state"],
+    ["text", "state"],
+    live=True,
+)
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()

gradio/components.py

@@ -16,7 +16,7 @@ import tempfile
 import warnings
 from copy import deepcopy
 from types import ModuleType
-from typing import Any, Dict, List, Optional, Tuple, Type
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type
 
 import matplotlib.figure
 import numpy
@@ -34,6 +34,7 @@ from gradio.events import (
     Clickable,
     Editable,
     Playable,
+    Streamable,
     Submittable,
 )
 
@@ -1306,7 +1307,7 @@ class Dropdown(Radio):
         )
 
 
-class Image(Editable, Clearable, Changeable, IOComponent):
+class Image(Editable, Clearable, Changeable, Streamable, IOComponent):
     """
     Creates an image component that can be used to upload/draw images (as an input) or display images (as an output).
     Preprocessing: passes the uploaded image as a {numpy.array}, {PIL.Image} or {str} filepath depending on `type`.
@@ -1330,6 +1331,7 @@ class Image(Editable, Clearable, Changeable, IOComponent):
         interactive: Optional[bool] = None,
         visible: bool = True,
         elem_id: Optional[str] = None,
+        streaming: bool = False,
         **kwargs,
     ):
         """
@@ -1344,6 +1346,7 @@ class Image(Editable, Clearable, Changeable, IOComponent):
         label (Optional[str]): component name in interface.
         show_label (bool): if True, will display label.
         visible (bool): If False, component will be hidden.
+        streaming (bool): If True when used in a `live` interface, will automatically stream webcam feed. Only valid is source is 'webcam'.
         """
         self.type = type
         self.value = (
@@ -1359,6 +1362,10 @@ class Image(Editable, Clearable, Changeable, IOComponent):
         self.invert_colors = invert_colors
         self.test_input = deepcopy(media_data.BASE64_IMAGE)
         self.interpret_by_tokens = True
+        self.streaming = streaming
+        if streaming and source != "webcam":
+            raise ValueError("Image streaming only available if source is 'webcam'.")
+
         IOComponent.__init__(
             self,
             label=label,
@@ -1377,6 +1384,7 @@ class Image(Editable, Clearable, Changeable, IOComponent):
             "source": self.source,
             "tool": self.tool,
             "value": self.value,
+            "streaming": self.streaming,
             **IOComponent.get_config(self),
         }
 
@@ -1635,6 +1643,25 @@ class Image(Editable, Clearable, Changeable, IOComponent):
             container_bg_color=container_bg_color,
         )
 
+    def stream(
+        self,
+        fn: Callable,
+        inputs: List[Component],
+        outputs: List[Component],
+        _js: Optional[str] = None,
+    ):
+        """
+        Parameters:
+            fn: Callable function
+            inputs: List of inputs
+            outputs: List of outputs
+            _js: Optional frontend js method to run before running 'fn'. Input arguments for js method are values of 'inputs' and 'outputs', return should be a list of values for output components.
+        Returns: None
+        """
+        if self.source != "webcam":
+            raise ValueError("Image streaming only available if source is 'webcam'.")
+        Streamable.stream(self, fn, inputs, outputs, _js)
+
 
 class Video(Changeable, Clearable, Playable, IOComponent):
     """
@@ -1777,7 +1804,7 @@ class Video(Changeable, Clearable, Playable, IOComponent):
         return processing_utils.decode_base64_to_file(x).name
 
 
-class Audio(Changeable, Clearable, Playable, IOComponent):
+class Audio(Changeable, Clearable, Playable, Streamable, IOComponent):
     """
     Creates an audio component that can be used to upload/record audio (as an input) or display audio (as an output).
     Preprocessing: passes the uploaded audio as a {Tuple(int, numpy.array)} corresponding to (sample rate, data) or as a {str} filepath, depending on `type`
@@ -1797,6 +1824,7 @@ class Audio(Changeable, Clearable, Playable, IOComponent):
         interactive: Optional[bool] = None,
         visible: bool = True,
         elem_id: Optional[str] = None,
+        streaming: bool = False,
         **kwargs,
     ):
         """
@@ -1807,6 +1835,7 @@ class Audio(Changeable, Clearable, Playable, IOComponent):
         label (Optional[str]): component name in interface.
         show_label (bool): if True, will display label.
         visible (bool): If False, component will be hidden.
+        streaming (bool): If set to true when used in a `live` interface, will automatically stream webcam feed. Only valid is source is 'microphone'.
         """
         self.value = (
             processing_utils.encode_url_or_file_to_base64(value) if value else None
@@ -1817,6 +1846,11 @@ class Audio(Changeable, Clearable, Playable, IOComponent):
         self.output_type = "auto"
         self.test_input = deepcopy(media_data.BASE64_AUDIO)
         self.interpret_by_tokens = True
+        self.streaming = streaming
+        if streaming and source != "microphone":
+            raise ValueError(
+                "Audio streaming only available if source is 'microphone'."
+            )
         IOComponent.__init__(
             self,
             label=label,
@@ -1832,6 +1866,7 @@ class Audio(Changeable, Clearable, Playable, IOComponent):
         return {
             "source": self.source,  # TODO: This did not exist in output template, careful here if an error arrives
             "value": self.value,
+            "streaming": self.streaming,
             **IOComponent.get_config(self),
         }
 
@@ -2056,6 +2091,27 @@ class Audio(Changeable, Clearable, Playable, IOComponent):
     def deserialize(self, x):
         return processing_utils.decode_base64_to_file(x).name
 
+    def stream(
+        self,
+        fn: Callable,
+        inputs: List[Component],
+        outputs: List[Component],
+        _js: Optional[str] = None,
+    ):
+        """
+        Parameters:
+            fn: Callable function
+            inputs: List of inputs
+            outputs: List of outputs
+            _js: Optional frontend js method to run before running 'fn'. Input arguments for js method are values of 'inputs' and 'outputs', return should be a list of values for output components.
+        Returns: None
+        """
+        if self.source != "microphone":
+            raise ValueError(
+                "Audio streaming only available if source is 'microphone'."
+            )
+        Streamable.stream(self, fn, inputs, outputs, _js)
+
 
 class File(Changeable, Clearable, IOComponent):
     """

gradio/events.py

@@ -179,3 +179,23 @@ class Playable(Block):
         Returns: None
         """
         self.set_event_trigger("stop", fn, inputs, outputs, js=_js)
+
+
+class Streamable(Block):
+    def stream(
+        self,
+        fn: Callable,
+        inputs: List[Component],
+        outputs: List[Component],
+        _js: Optional[str] = None,
+    ):
+        """
+        Parameters:
+            fn: Callable function
+            inputs: List of inputs
+            outputs: List of outputs
+            _js: Optional frontend js method to run before running 'fn'. Input arguments for js method are values of 'inputs' and 'outputs', return should be a list of values for output components.
+        Returns: None
+        """
+        self.streaming = True
+        self.set_event_trigger("stream", fn, inputs, outputs, js=_js)

gradio/interface.py

@@ -33,6 +33,7 @@ from gradio.components import (
     Variable,
     get_component_instance,
 )
+from gradio.events import Changeable, Streamable
 from gradio.external import load_from_pipeline, load_interface  # type: ignore
 from gradio.flagging import CSVLogger, FlaggingCallback  # type: ignore
 from gradio.layouts import Column, Row, TabItem, Tabs
@@ -502,9 +503,22 @@ class Interface(Blocks):
             )
             if self.live:
                 for component in self.input_components:
-                    component.change(
-                        submit_fn, self.input_components, self.output_components
-                    )
+                    if isinstance(component, Streamable):
+                        if component.streaming:
+                            component.stream(
+                                submit_fn, self.input_components, self.output_components
+                            )
+                            continue
+                        else:
+                            print(
+                                "Hint: Set streaming=True for "
+                                + component.__class__.__name__
+                                + " component to use live streaming."
+                            )
+                    if isinstance(component, Changeable):
+                        component.change(
+                            submit_fn, self.input_components, self.output_components
+                        )
             else:
                 submit_btn.click(
                     submit_fn,

gradio/test_data/blocks_configs.py

@@ -44,6 +44,7 @@ XRAY_CONFIG = {
                 "tool": "editor",
                 "show_label": True,
                 "name": "image",
+                "streaming": False,
                 "visible": True,
                 "style": {},
             },
@@ -89,6 +90,7 @@ XRAY_CONFIG = {
                 "tool": "editor",
                 "show_label": True,
                 "name": "image",
+                "streaming": False,
                 "visible": True,
                 "style": {},
             },
@@ -239,6 +241,7 @@ XRAY_CONFIG_DIFF_IDS = {
                 "tool": "editor",
                 "show_label": True,
                 "name": "image",
+                "streaming": False,
                 "visible": True,
                 "style": {},
             },
@@ -284,6 +287,7 @@ XRAY_CONFIG_DIFF_IDS = {
                 "tool": "editor",
                 "show_label": True,
                 "name": "image",
+                "streaming": False,
                 "visible": True,
                 "style": {},
             },
@@ -439,6 +443,7 @@ XRAY_CONFIG_WITH_MISTAKE = {
                 "source": "upload",
                 "tool": "editor",
                 "name": "image",
+                "streaming": False,
                 "style": {},
             },
         },
@@ -484,6 +489,7 @@ XRAY_CONFIG_WITH_MISTAKE = {
                 "source": "upload",
                 "tool": "editor",
                 "name": "image",
+                "streaming": False,
                 "style": {},
             },
         },

test/test_components.py

@@ -577,6 +577,7 @@ class TestImage(unittest.TestCase):
                 "source": "upload",
                 "tool": "editor",
                 "name": "image",
+                "streaming": False,
                 "show_label": True,
                 "label": "Upload Your Image",
                 "style": {},
@@ -734,6 +735,7 @@ class TestAudio(unittest.TestCase):
             {
                 "source": "upload",
                 "name": "audio",
+                "streaming": False,
                 "show_label": True,
                 "label": "Upload Your Audio",
                 "style": {},
@@ -776,6 +778,7 @@ class TestAudio(unittest.TestCase):
             audio_output.get_config(),
             {
                 "name": "audio",
+                "streaming": False,
                 "show_label": True,
                 "label": None,
                 "source": "upload",

ui/packages/app/src/Blocks.svelte

@@ -178,7 +178,6 @@
 	}
 
 	let handled_dependencies: Array<number[]> = [];
-	let status_tracker_values: Record<number, string> = {};
 
 	async function handle_mount() {
 		await tick();
@@ -311,7 +310,7 @@
 	$: set_status($loading_status);
 
 	dependencies.forEach((v, i) => {
-		loading_status.register(i, v.outputs);
+		loading_status.register(i, v.inputs, v.outputs);
 	});
 
 	function set_status(
@@ -320,6 +319,10 @@
 		for (const id in statuses) {
 			set_prop(instance_map[id], "loading_status", statuses[id]);
 		}
+		const inputs_to_update = loading_status.get_inputs_to_update();
+		for (const [id, pending_status] of inputs_to_update) {
+			set_prop(instance_map[id], "pending", pending_status === "pending");
+		}
 	}
 
 	let mode = "";
@@ -387,7 +390,6 @@
 				{instance_map}
 				{theme}
 				{root}
-				{status_tracker_values}
 				on:mount={handle_mount}
 				on:destroy={({ detail }) => handle_destroy(detail)}
 			/>

ui/packages/app/src/Render.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
 	import { onMount, createEventDispatcher, setContext } from "svelte";
+	import { loading_status } from "./stores";
 
 	export let root: string;
 	export let component;
@@ -105,7 +106,6 @@
 	elem_id={props.elem_id || id}
 	{...props}
 	{root}
-	tracked_status={status_tracker_values[id]}
 >
 	{#if children && children.length}
 		{#each children as { component, id: each_id, props, children, has_modes } (each_id)}
@@ -119,7 +119,6 @@
 				{children}
 				{dynamic_ids}
 				{has_modes}
-				{status_tracker_values}
 				on:destroy
 				on:mount
 			/>

ui/packages/app/src/components/Audio/Audio.svelte

@@ -6,6 +6,11 @@
 
 	import StatusTracker from "../StatusTracker/StatusTracker.svelte";
 	import type { LoadingStatus } from "../StatusTracker/types";
+	import { createEventDispatcher } from "svelte";
+	const dispatch = createEventDispatcher<{
+		change;
+		stream;
+	}>();
 
 	import { _ } from "svelte-i18n";
 
@@ -18,6 +23,8 @@
 	export let label: string;
 	export let root: string;
 	export let show_label: boolean;
+	export let pending: boolean;
+	export let streaming: boolean;
 
 	export let loading_status: LoadingStatus;
 
@@ -42,11 +49,20 @@
 			{label}
 			{show_label}
 			value={_value}
-			on:change={({ detail }) => (value = detail)}
+			on:change={({ detail }) => {
+				value = detail;
+				dispatch("change", value);
+			}}
+			on:stream={({ detail }) => {
+				value = detail;
+				dispatch("stream", value);
+			}}
 			on:drag={({ detail }) => (dragging = detail)}
 			{name}
 			{source}
 			{type}
+			{pending}
+			{streaming}
 			on:edit
 			on:play
 			on:pause

ui/packages/app/src/components/Image/Image.svelte

@@ -11,6 +11,8 @@
 	export let tool: "editor" | "select" = "editor";
 	export let label: string;
 	export let show_label: boolean;
+	export let streaming: boolean;
+	export let pending: boolean;
 
 	export let loading_status: LoadingStatus;
 
@@ -42,9 +44,12 @@
 			on:edit
 			on:clear
 			on:change
+			on:stream
 			on:drag={({ detail }) => (dragging = detail)}
 			{label}
 			{show_label}
+			{pending}
+			{streaming}
 			drop_text={$_("interface.drop_image")}
 			or_text={$_("or")}
 			upload_text={$_("interface.click_to_upload")}

ui/packages/app/src/stores.ts

@@ -10,8 +10,12 @@ export interface LoadingStatus {
 function create_loading_status_store() {
 	const store = writable<Record<string, Omit<LoadingStatus, "outputs">>>({});
 
+	const fn_inputs: Array<Array<number>> = [];
 	const fn_outputs: Array<Array<number>> = [];
 	const pending_outputs = new Map<number, number>();
+	const pending_inputs = new Map<number, number>();
+
+	const inputs_to_update = new Map<number, string>();
 	const fn_status: Array<LoadingStatus["status"]> = [];
 
 	function update(
@@ -21,6 +25,7 @@ function create_loading_status_store() {
 		eta: LoadingStatus["eta"]
 	) {
 		const outputs = fn_outputs[fn_index];
+		const inputs = fn_inputs[fn_index];
 		const last_status = fn_status[fn_index];
 
 		const outputs_to_update = outputs.map((id) => {
@@ -56,6 +61,22 @@ function create_loading_status_store() {
 			};
 		});
 
+		inputs.map((id) => {
+			const pending_count = pending_inputs.get(id) || 0;
+
+			// from (pending -> error) | complete - decrement pending count
+			if (last_status === "pending" && status !== "pending") {
+				let new_count = pending_count - 1;
+				pending_inputs.set(id, new_count < 0 ? 0 : new_count);
+				inputs_to_update.set(id, status);
+			} else if (last_status !== "pending" && status === "pending") {
+				pending_inputs.set(id, pending_count + 1);
+				inputs_to_update.set(id, status);
+			} else {
+				inputs_to_update.delete(id);
+			}
+		});
+
 		store.update((outputs) => {
 			outputs_to_update.forEach(({ id, queue_position, eta, status }) => {
 				outputs[id] = {
@@ -71,7 +92,12 @@ function create_loading_status_store() {
 		fn_status[fn_index] = status;
 	}
 
-	function register(index: number, outputs: Array<number>) {
+	function register(
+		index: number,
+		inputs: Array<number>,
+		outputs: Array<number>
+	) {
+		fn_inputs[index] = inputs;
 		fn_outputs[index] = outputs;
 	}
 
@@ -81,6 +107,9 @@ function create_loading_status_store() {
 		subscribe: store.subscribe,
 		get_status_for_fn(i: number) {
 			return fn_status[i];
+		},
+		get_inputs_to_update() {
+			return inputs_to_update;
 		}
 	};
 }

ui/packages/audio/src/Audio.svelte

@@ -19,6 +19,8 @@
 	export let show_label: boolean;
 	export let name: string;
 	export let source: "microphone" | "upload" | "none";
+	export let pending: boolean = false;
+	export let streaming: boolean = false;
 	export let drop_text: string = "Drop an audio file";
 	export let or_text: string = "or";
 	export let upload_text: string = "click to upload";
@@ -37,6 +39,7 @@
 
 	const dispatch = createEventDispatcher<{
 		change: AudioData;
+		stream: AudioData;
 		edit: AudioData;
 		play: undefined;
 		pause: undefined;
@@ -62,16 +65,15 @@
 		});
 
 		recorder.addEventListener("stop", async () => {
-			recording = false;
+			if (!streaming) {
+				recording = false;
+			}
 			audio_blob = new Blob(audio_chunks, { type: "audio/wav" });
 			value = {
 				data: await blob_to_data_url(audio_blob),
 				name
 			};
-			dispatch("change", {
-				data: await blob_to_data_url(audio_blob),
-				name
-			});
+			dispatch(streaming ? "stream" : "change", value);
 		});
 	}
 
@@ -92,6 +94,9 @@
 
 	const stop = () => {
 		recorder.stop();
+		if (streaming) {
+			recording = false;
+		}
 	};
 
 	function clear() {
@@ -149,10 +154,24 @@
 
 	export let dragging = false;
 	$: dispatch("drag", dragging);
+
+	if (streaming) {
+		window.setInterval(() => {
+			if (
+				recording &&
+				recorder &&
+				recorder.state === "recording" &&
+				pending === false
+			) {
+				stop();
+				record();
+			}
+		}, 500);
+	}
 </script>
 
 <BlockLabel {show_label} Icon={Music} label={label || "Audio"} />
-{#if value === null}
+{#if value === null || streaming}
 	{#if source === "microphone"}
 		<div class="mt-6 p-2">
 			{#if recording}

ui/packages/image/src/Image.svelte

@@ -21,6 +21,8 @@
 	export let drop_text: string = "Drop an image file";
 	export let or_text: string = "or";
 	export let upload_text: string = "click to upload";
+	export let streaming: boolean = false;
+	export let pending: boolean = false;
 
 	let mode: "edit" | "view" = "view";
 	let sketch: Sketch;
@@ -37,11 +39,12 @@
 	function handle_save({ detail }: { detail: string }) {
 		value = detail;
 		mode = "view";
-		dispatch("edit");
+		dispatch(streaming ? "stream" : "edit");
 	}
 
 	const dispatch = createEventDispatcher<{
 		change: string | null;
+		stream: string | null;
 		edit: undefined;
 		clear: undefined;
 		drag: boolean;
@@ -67,7 +70,7 @@
 			on:clear={() => sketch.clear()}
 		/>
 		<Sketch {value} bind:this={sketch} on:change={handle_save} />
-	{:else if value === null}
+	{:else if value === null || streaming}
 		{#if source === "upload"}
 			<Upload
 				bind:dragging
@@ -82,7 +85,12 @@
 				</div>
 			</Upload>
 		{:else if source === "webcam"}
-			<Webcam on:capture={handle_save} />
+			<Webcam
+				on:capture={handle_save}
+				on:stream={handle_save}
+				{streaming}
+				{pending}
+			/>
 		{/if}
 	{:else if tool === "select"}
 		<Cropper image={value} on:crop={handle_save} />

ui/packages/image/src/Webcam.svelte

@@ -4,6 +4,8 @@
 
 	let video_source: HTMLVideoElement;
 	let canvas: HTMLCanvasElement;
+	export let streaming: boolean = false;
+	export let pending: boolean = false;
 
 	export let mode: "image" | "video" = "image";
 
@@ -38,7 +40,7 @@
 			);
 
 			var data = canvas.toDataURL("image/png");
-			dispatch("capture", data);
+			dispatch(streaming ? "stream" : "capture", data);
 		}
 	}
 
@@ -88,29 +90,39 @@
 	}
 
 	access_webcam();
+
+	if (streaming && mode === "image") {
+		window.setInterval(() => {
+			if (video_source && !pending) {
+				take_picture();
+			}
+		}, 500);
+	}
 </script>
 
 <div class="h-full min-h-[15rem] w-full relative">
 	<!-- svelte-ignore a11y-media-has-caption -->
 	<video bind:this={video_source} class="h-full w-full " />
-	<button
-		on:click={mode === "image" ? take_picture : take_recording}
-		class="rounded-xl w-10 h-10 flex justify-center items-center absolute inset-x-0 bottom-2 md:bottom-4 xl:bottom-8 m-auto drop-shadow-lg bg-black/90"
-	>
-		{#if mode === "video"}
-			{#if recording}
-				<div class="w-2/4 h-2/4 dark:text-white opacity-80">
-					<Square />
-				</div>
+	{#if !streaming}
+		<button
+			on:click={mode === "image" ? take_picture : take_recording}
+			class="rounded-xl w-10 h-10 flex justify-center items-center absolute inset-x-0 bottom-2 md:bottom-4 xl:bottom-8 m-auto drop-shadow-lg bg-black/90"
+		>
+			{#if mode === "video"}
+				{#if recording}
+					<div class="w-2/4 h-2/4 dark:text-white opacity-80">
+						<Square />
+					</div>
+				{:else}
+					<div class="w-2/4 h-2/4 dark:text-white opacity-80">
+						<Circle />
+					</div>
+				{/if}
 			{:else}
 				<div class="w-2/4 h-2/4 dark:text-white opacity-80">
-					<Circle />
+					<Camera />
 				</div>
 			{/if}
-		{:else}
-			<div class="w-2/4 h-2/4 dark:text-white opacity-80">
-				<Camera />
-			</div>
-		{/if}
-	</button>
+		</button>
+	{/if}
 </div>