Fix audio recording events not dispatching (#7158)

* fix audio events and add events test

* add changeset

* add changes to audio demo

* Fix e2e tests

* regenerate notebook

* formatting

* test timeout for e2e

* formatting

* remove value param from stop_recording

* formatting

* fix test

* formatting

* tweak

* test new playwright config

* skip test

* formatting

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

.changeset/every-bears-peel.md

@@ -0,0 +1,6 @@
+---
+"@gradio/audio": patch
+"gradio": patch
+---
+
+fix:Fix audio recording events not dispatching

.config/playwright.config.js

@@ -10,7 +10,8 @@ export default defineConfig({
 			args: [
 				"--disable-web-security",
 				"--use-fake-device-for-media-stream",
-				"--use-fake-ui-for-media-stream"
+				"--use-fake-ui-for-media-stream",
+				"--use-file-for-fake-audio-capture=../gradio/test_data/test_audio.wav"
 			]
 		}
 	},

demo/audio_component_events/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: audio_component_events"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", "    with gr.Row():\n", "        with gr.Column():\n", "            input_video = gr.Audio(type=\"filepath\", label=\"Input Audio\", sources=[\"upload\", \"microphone\"])\n", "        with gr.Column():\n", "            output_video = gr.Audio(label=\"Output Audio\", sources=[\"upload\", \"microphone\"])\n", "\n", "    with gr.Row():\n", "        with gr.Column():\n", "            input_num_change = gr.Number(label=\"# Input Change Events\", value=0)\n", "            input_num_load = gr.Number(label=\"# Input Upload Events\", value=0)\n", "            input_num_play = gr.Number(label=\"# Input Play Events\", value=0)\n", "            input_num_pause = gr.Number(label=\"# Input Pause Events\", value=0)\n", "\n", "        with gr.Column():\n", "            output_num_play = gr.Number(label=\"# Output Play Events\", value=0)\n", "            output_num_pause = gr.Number(label=\"# Output Pause Events\", value=0)\n", "            output_num_stop = gr.Number(label=\"# Output Stop Events\", value=0)\n", "\n", "            input_video.upload(lambda s, n: (s, n + 1), [input_video, input_num_load], [output_video, input_num_load])\n", "            input_video.change(lambda n: n + 1, input_num_change, input_num_change)\n", "            input_video.play(lambda n: n + 1, input_num_play, input_num_play)\n", "            input_video.pause(lambda n: n + 1, input_num_pause, input_num_pause)\n", "            input_video.change(lambda n: n + 1, input_num_change, input_num_change)\n", "\n", "            output_video.play(lambda n: n + 1, output_num_play, output_num_play)\n", "            output_video.pause(lambda n: n + 1, output_num_pause, output_num_pause)\n", "            output_video.stop(lambda n: n + 1, output_num_stop, output_num_stop)\n", "\n", "\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: audio_component_events"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "with gr.Blocks() as demo:\n", "    with gr.Row():\n", "        with gr.Column():\n", "            input_video = gr.Audio(type=\"filepath\", label=\"Input Audio\", sources=[\"upload\", \"microphone\"])\n", "        with gr.Column():\n", "            output_video = gr.Audio(label=\"Output Audio\", sources=[\"upload\", \"microphone\"])\n", "\n", "    with gr.Row():\n", "        with gr.Column():\n", "            input_num_change = gr.Number(label=\"# Input Change Events\", value=0)\n", "            input_num_load = gr.Number(label=\"# Input Upload Events\", value=0)\n", "            input_num_play = gr.Number(label=\"# Input Play Events\", value=0)\n", "            input_num_pause = gr.Number(label=\"# Input Pause Events\", value=0)\n", "\n", "        with gr.Column():\n", "            input_record = gr.Number(label=\"# Input Start Recording Events\", value=0)\n", "            input_pause = gr.Number(label=\"# Input Pause Recording Events\", value=0)\n", "            input_stop = gr.Number(label=\"# Input Stop Recording Events\", value=0)\n", "\n", "\n", "        with gr.Column():\n", "            output_num_play = gr.Number(label=\"# Output Play Events\", value=0)\n", "            output_num_pause = gr.Number(label=\"# Output Pause Events\", value=0)\n", "            output_num_stop = gr.Number(label=\"# Output Stop Events\", value=0)\n", "\n", "            input_video.upload(lambda s, n: (s, n + 1), [input_video, input_num_load], [output_video, input_num_load])\n", "            input_video.change(lambda n: n + 1, input_num_change, input_num_change)\n", "            input_video.play(lambda n: n + 1, input_num_play, input_num_play)\n", "            input_video.pause(lambda n: n + 1, input_num_pause, input_num_pause)\n", "            input_video.change(lambda n: n + 1, input_num_change, input_num_change)\n", "\n", "            input_video.start_recording(lambda n: n + 1, input_record, input_record)\n", "            input_video.pause_recording(lambda n: n + 1, input_pause, input_pause)\n", "            input_video.stop_recording(lambda n: n + 1, input_stop, input_stop)\n", "\n", "            output_video.play(lambda n: n + 1, output_num_play, output_num_play)\n", "            output_video.pause(lambda n: n + 1, output_num_pause, output_num_pause)\n", "            output_video.stop(lambda n: n + 1, output_num_stop, output_num_stop)\n", "\n", "\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/audio_component_events/run.py

@@ -14,6 +14,12 @@ with gr.Blocks() as demo:
             input_num_play = gr.Number(label="# Input Play Events", value=0)
             input_num_pause = gr.Number(label="# Input Pause Events", value=0)
 
+        with gr.Column():
+            input_record = gr.Number(label="# Input Start Recording Events", value=0)
+            input_pause = gr.Number(label="# Input Pause Recording Events", value=0)
+            input_stop = gr.Number(label="# Input Stop Recording Events", value=0)
+
+
         with gr.Column():
             output_num_play = gr.Number(label="# Output Play Events", value=0)
             output_num_pause = gr.Number(label="# Output Pause Events", value=0)
@@ -25,6 +31,10 @@ with gr.Blocks() as demo:
             input_video.pause(lambda n: n + 1, input_num_pause, input_num_pause)
             input_video.change(lambda n: n + 1, input_num_change, input_num_change)
 
+            input_video.start_recording(lambda n: n + 1, input_record, input_record)
+            input_video.pause_recording(lambda n: n + 1, input_pause, input_pause)
+            input_video.stop_recording(lambda n: n + 1, input_stop, input_stop)
+
             output_video.play(lambda n: n + 1, output_num_play, output_num_play)
             output_video.pause(lambda n: n + 1, output_num_pause, output_num_pause)
             output_video.stop(lambda n: n + 1, output_num_stop, output_num_stop)

js/app/test/audio_component_events.spec.ts

@@ -1,4 +1,5 @@
 import { test, expect, drag_and_drop_file } from "@gradio/tootils";
+import { chromium } from "playwright";
 
 test("Audio click-to-upload uploads audio successfuly. File downloading works and file has correct name.", async ({
 	page
@@ -79,3 +80,36 @@ test.skip("Play, Pause, and stop events work correctly.", async ({ page }) => {
 		.click();
 	await expect(async () => event_triggered("# Output Stop Events")).toPass();
 });
+
+test.skip("Record, pause, and stop recording events work correctly.", async ({
+	page
+}) => {
+	const browser = await chromium.launch();
+	const context = await browser.newContext({
+		permissions: ["microphone"]
+	});
+	context.grantPermissions(["microphone"]);
+
+	await page.getByLabel("Record audio").click();
+	await page.getByRole("button", { name: "Record", exact: true }).click();
+
+	expect(await page.getByLabel("# Input Start Recording Events")).toHaveValue(
+		"1"
+	);
+
+	await page.waitForTimeout(2000);
+	await page.getByLabel("pause", { exact: true }).click();
+	await page.getByRole("button", { name: "Resume" }).click();
+
+	expect(await page.getByLabel("# Input Pause Recording Events")).toHaveValue(
+		"1"
+	);
+
+	await page.getByRole("button", { name: "Stop" }).click();
+
+	expect(await page.getByLabel("# Input Stop Recording Events")).toHaveValue(
+		"1"
+	);
+
+	expect(await page.getByLabel("# Input Change Events")).toHaveValue("1");
+});

js/app/test/blocks_flashcards.spec.ts

@@ -43,12 +43,4 @@ test("shows the results tab when results > 0", async ({ page }) => {
 
 	await page.waitForTimeout(1000);
 	await page.getByText("Start Practice").dblclick();
-
-	await page.waitForTimeout(5000);
-
-	// await page.getByRole("button", { name: "New Card" }).click();
-	// await page.waitForTimeout(1000);
-	// await page.getByRole("button", { name: "Flip Card" }).click();
-	// await page.getByRole("button", { name: "Correct", exact: true }).click();
-	// await page.getByRole("tab", { name: "Results" }).click();
 });

js/audio/Index.svelte

@@ -202,7 +202,7 @@
 			on:stop={() => gradio.dispatch("stop")}
 			on:start_recording={() => gradio.dispatch("start_recording")}
 			on:pause_recording={() => gradio.dispatch("pause_recording")}
-			on:stop_recording={(e) => gradio.dispatch("stop_recording", e.detail)}
+			on:stop_recording={(e) => gradio.dispatch("stop_recording")}
 			on:upload={() => gradio.dispatch("upload")}
 			on:clear={() => gradio.dispatch("clear")}
 			on:error={handle_error}

js/audio/interactive/InteractiveAudio.svelte

@@ -220,67 +220,81 @@
 	float={active_source === "upload" && value === null}
 	label={label || i18n("audio.audio")}
 />
-{#if value === null || streaming}
-	{#if active_source === "microphone"}
-		<ModifyUpload {i18n} on:clear={clear} absolute={true} />
-		{#if streaming}
-			<StreamAudio
-				{record}
-				{recording}
-				{stop}
-				{i18n}
-				{waveform_settings}
-				{waveform_options}
-			/>
-		{:else}
-			<AudioRecorder
-				bind:mode
-				{i18n}
-				{editable}
-				{dispatch_blob}
-				{waveform_settings}
-				{waveform_options}
-				{handle_reset_value}
-			/>
+<div class="audio-container">
+	{#if value === null || streaming}
+		{#if active_source === "microphone"}
+			<ModifyUpload {i18n} on:clear={clear} absolute={true} />
+			{#if streaming}
+				<StreamAudio
+					{record}
+					{recording}
+					{stop}
+					{i18n}
+					{waveform_settings}
+					{waveform_options}
+				/>
+			{:else}
+				<AudioRecorder
+					bind:mode
+					{i18n}
+					{editable}
+					{dispatch_blob}
+					{waveform_settings}
+					{waveform_options}
+					{handle_reset_value}
+					on:start_recording
+					on:pause_recording
+					on:stop_recording
+				/>
+			{/if}
+		{:else if active_source === "upload"}
+			<!-- explicitly listed out audio mimetypes due to iOS bug not recognizing audio/* -->
+			<Upload
+				filetype="audio/aac,audio/midi,audio/mpeg,audio/ogg,audio/wav,audio/x-wav,audio/opus,audio/webm,audio/flac,audio/vnd.rn-realaudio,audio/x-ms-wma,audio/x-aiff,audio/amr,audio/*"
+				on:load={handle_load}
+				bind:dragging
+				on:error={({ detail }) => dispatch("error", detail)}
+				{root}
+			>
+				<slot />
+			</Upload>
 		{/if}
-	{:else if active_source === "upload"}
-		<!-- explicitly listed out audio mimetypes due to iOS bug not recognizing audio/* -->
-		<Upload
-			filetype="audio/aac,audio/midi,audio/mpeg,audio/ogg,audio/wav,audio/x-wav,audio/opus,audio/webm,audio/flac,audio/vnd.rn-realaudio,audio/x-ms-wma,audio/x-aiff,audio/amr,audio/*"
-			on:load={handle_load}
-			bind:dragging
-			on:error={({ detail }) => dispatch("error", detail)}
-			{root}
-		>
-			<slot />
-		</Upload>
+	{:else}
+		<ModifyUpload
+			{i18n}
+			on:clear={clear}
+			on:edit={() => (mode = "edit")}
+			download={show_download_button ? value.url : null}
+			absolute={true}
+		/>
+
+		<AudioPlayer
+			bind:mode
+			{value}
+			{label}
+			{i18n}
+			{dispatch_blob}
+			{waveform_settings}
+			{waveform_options}
+			{trim_region_settings}
+			{handle_reset_value}
+			{editable}
+			interactive
+			on:stop
+			on:play
+			on:pause
+			on:edit
+		/>
 	{/if}
-{:else}
-	<ModifyUpload
-		{i18n}
-		on:clear={clear}
-		on:edit={() => (mode = "edit")}
-		download={show_download_button ? value.url : null}
-		absolute={true}
-	/>
 
-	<AudioPlayer
-		bind:mode
-		{value}
-		{label}
-		{i18n}
-		{dispatch_blob}
-		{waveform_settings}
-		{waveform_options}
-		{trim_region_settings}
-		{handle_reset_value}
-		{editable}
-		interactive
-		on:stop
-		on:play
-		on:pause
-		on:edit
-	/>
-{/if}
+	<SelectSource {sources} bind:active_source handle_clear={clear} />
+</div>
 
-<SelectSource {sources} bind:active_source handle_clear={clear} />
+<style>
+	.audio-container {
+		height: calc(var(--size-full) - var(--size-6));
+		display: flex;
+		flex-direction: column;
+		justify-content: space-between;
+	}
+</style>

js/audio/player/AudioPlayer.svelte

@@ -201,6 +201,7 @@
 <style>
 	.component-wrapper {
 		padding: var(--size-3);
+		width: 100%;
 	}
 
 	:global(::part(wrapper)) {

js/audio/recorder/AudioRecorder.svelte

@@ -82,17 +82,21 @@
 		seconds = 0;
 		timing = false;
 		clearInterval(interval);
-		const array_buffer = await blob.arrayBuffer();
-		const context = new AudioContext({
-			sampleRate: waveform_settings.sampleRate
-		});
-		const audio_buffer = await context.decodeAudioData(array_buffer);
-
-		if (audio_buffer)
-			await process_audio(audio_buffer).then(async (audio: Uint8Array) => {
-				await dispatch_blob([audio], "change");
-				await dispatch_blob([audio], "stop_recording");
+		try {
+			const array_buffer = await blob.arrayBuffer();
+			const context = new AudioContext({
+				sampleRate: waveform_settings.sampleRate
 			});
+			const audio_buffer = await context.decodeAudioData(array_buffer);
+
+			if (audio_buffer)
+				await process_audio(audio_buffer).then(async (audio: Uint8Array) => {
+					await dispatch_blob([audio], "change");
+					await dispatch_blob([audio], "stop_recording");
+				});
+		} catch (e) {
+			console.error(e);
+		}
 	});
 
 	$: record?.on("record-pause", () => {
@@ -259,6 +263,7 @@
 
 	.component-wrapper {
 		padding: var(--size-3);
+		width: 100%;
 	}
 
 	.timestamps {

js/audio/shared/WaveformRecordControls.svelte

@@ -91,6 +91,7 @@
 		>
 
 		<button
+			aria-label="pause"
 			bind:this={pauseButton}
 			class="pause-button"
 			on:click={() => record.pauseRecording()}><Pause /></button