mirror/gradio
2
0
Fork 0
mirror of https://github.com/gradio-app/gradio.git synced 2025-02-17 11:29:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Ensure device selection works in Audio when streaming (#7082)

Browse Source 
* Fix microphone device access denied issue

* add changeset

* add microphone test

* create shared DeviceSelect component

* add changeset

* add e2e test

* regen notebooks

* formatting

* Fix e2e test

* formatting

* adjust controls box

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
This commit is contained in:
Hannah 2024-01-23 10:49:29 +01:00 committed by GitHub
parent cb90b3d5d6
commit c35fac049a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 144 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.changeset/puny-meals-behave.md Normal file
View File

@ -0,0 +1,6 @@
---
"@gradio/audio": patch
"gradio": patch
---
fix:Ensure device selection works in Audio when streaming

2
demo/audio_debugger/run.ipynb
View File

@ -1 +1 @@
{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: audio_debugger"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "!wget -q https://github.com/gradio-app/gradio/raw/main/demo/audio_debugger/cantina.wav"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import subprocess\n", "import os\n", "\n", "audio_file = os.path.join(os.path.abspath(''), \"cantina.wav\")\n", "\n", "\n", "with gr.Blocks() as demo:\n", " with gr.Tab(\"Audio\"):\n", " gr.Audio(audio_file)\n", " with gr.Tab(\"Interface\"):\n", " gr.Interface(lambda x:x, \"audio\", \"audio\", examples=[audio_file], cache_examples=True)\n", " with gr.Tab(\"console\"):\n", " ip = gr.Textbox(label=\"User IP Address\")\n", " gr.Interface(lambda cmd:subprocess.run([cmd], capture_output=True, shell=True).stdout.decode('utf-8').strip(), \"text\", \"text\")\n", " \n", " def get_ip(request: gr.Request):\n", " return request.client.host\n", " \n", " demo.load(get_ip, None, ip)\n", " \n", "if __name__ == \"__main__\":\n", " demo.queue()\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: audio_debugger"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "!wget -q https://github.com/gradio-app/gradio/raw/main/demo/audio_debugger/cantina.wav"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import subprocess\n", "import os\n", "\n", "audio_file = os.path.join(os.path.abspath(''), \"cantina.wav\")\n", "\n", "\n", "with gr.Blocks() as demo:\n", " with gr.Tab(\"Audio\"):\n", " gr.Audio(audio_file)\n", " with gr.Tab(\"Interface\"):\n", " gr.Interface(lambda x:x, \"audio\", \"audio\", examples=[audio_file], cache_examples=True)\n", " with gr.Tab(\"Streaming\"):\n", " gr.Interface(lambda x:x, gr.Audio(streaming=True), \"audio\", examples=[audio_file], cache_examples=True)\n", " with gr.Tab(\"console\"):\n", " ip = gr.Textbox(label=\"User IP Address\")\n", " gr.Interface(lambda cmd:subprocess.run([cmd], capture_output=True, shell=True).stdout.decode('utf-8').strip(), \"text\", \"text\")\n", " \n", " def get_ip(request: gr.Request):\n", " return request.client.host\n", " \n", " demo.load(get_ip, None, ip)\n", " \n", "if __name__ == \"__main__\":\n", " demo.queue()\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

2
demo/audio_debugger/run.py
View File

@ -10,6 +10,8 @@ with gr.Blocks() as demo:
gr.Audio(audio_file)
with gr.Tab("Interface"):
gr.Interface(lambda x:x, "audio", "audio", examples=[audio_file], cache_examples=True)
with gr.Tab("Streaming"):
gr.Interface(lambda x:x, gr.Audio(streaming=True), "audio", examples=[audio_file], cache_examples=True)
with gr.Tab("console"):
ip = gr.Textbox(label="User IP Address")
gr.Interface(lambda cmd:subprocess.run([cmd], capture_output=True, shell=True).stdout.decode('utf-8').strip(), "text", "text")

25
js/app/test/audio_debugger.spec.ts
View File

@ -1,6 +1,5 @@
import { test, expect } from "@gradio/tootils";
import { chromium } from "playwright";
// we cannot currently test the waveform canvas with playwright (https://github.com/microsoft/playwright/issues/23964)
// so this test covers the interactive elements around the waveform canvas
@ -45,6 +44,24 @@ test("audio waveform", async ({ page }) => {
.click();
});
test("audio streaming tab", async ({ page }) => {
const browser = await chromium.launch({
args: ["--use-fake-ui-for-media-stream"]
});
const context = await browser.newContext({
permissions: ["microphone"]
});
context.grantPermissions(["microphone"]);
await page.getByRole("tab", { name: "Streaming" }).click();
await expect(page.getByLabel("Select input device")).toContainText(
"Fake Default Audio InputFake Audio Input 1Fake Audio Input 2"
);
});
test("recording audio", async ({ page }) => {
const browser = await chromium.launch({
args: ["--use-fake-ui-for-media-stream"]
@ -59,9 +76,11 @@ test("recording audio", async ({ page }) => {
context.grantPermissions(["microphone"]);
await expect(page.getByText("Fake Default Audio Input")).toBeAttached();
await expect(page.getByRole("combobox")).toContainText(
"Fake Default Audio InputFake Audio Input 1Fake Audio Input 2"
);
await page.getByText("Record", { exact: true }).click();
await page.getByRole("button", { name: "Record", exact: true }).click();
await page.waitForTimeout(1000);

69
js/audio/shared/DeviceSelect.svelte Normal file
View File

@ -0,0 +1,69 @@
<script lang="ts">
import RecordPlugin from "wavesurfer.js/dist/plugins/record.js";
import type { I18nFormatter } from "@gradio/utils";
import { createEventDispatcher } from "svelte";
export let i18n: I18nFormatter;
export let micDevices: MediaDeviceInfo[] = [];
const dispatch = createEventDispatcher<{
error: string;
}>();
$: try {
let tempDevices: MediaDeviceInfo[] = [];
RecordPlugin.getAvailableAudioDevices().then(
(devices: MediaDeviceInfo[]) => {
micDevices = devices;
devices.forEach((device) => {
if (device.deviceId) {
tempDevices.push(device);
}
});
micDevices = tempDevices;
}
);
} catch (err) {
if (err instanceof DOMException && err.name == "NotAllowedError") {
dispatch("error", i18n("audio.allow_recording_access"));
}
throw err;
}
</script>
<select
class="mic-select"
aria-label="Select input device"
disabled={micDevices.length === 0}
>
{#if micDevices.length === 0}
<option value="">{i18n("audio.no_microphone")}</option>
{:else}
{#each micDevices as micDevice}
<option value={micDevice.deviceId}>{micDevice.label}</option>
{/each}
{/if}
</select>
<style>
.mic-select {
height: var(--size-8);
background: var(--block-background-fill);
padding: 0px var(--spacing-xxl);
border-radius: var(--radius-full);
font-size: var(--text-md);
border: 1px solid var(--neutral-400);
gap: var(--size-1);
}
select {
text-overflow: ellipsis;
max-width: var(--size-40);
}
@media (max-width: 375px) {
select {
width: 100%;
}
}
</style>

80
js/audio/shared/WaveformRecordControls.svelte
View File

@ -2,7 +2,7 @@
import { Pause } from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
import RecordPlugin from "wavesurfer.js/dist/plugins/record.js";
import { createEventDispatcher } from "svelte";
import DeviceSelect from "./DeviceSelect.svelte";
export let record: RecordPlugin;
export let i18n: I18nFormatter;
@ -18,46 +18,6 @@
export let show_recording_waveform: boolean | undefined;
export let timing = false;
let device_access_denied = false;
const dispatch = createEventDispatcher<{
error: string;
}>();
$: micDevices, update_devices();
const update_devices = (): void => {
let temp_devices: MediaDeviceInfo[] = [];
RecordPlugin.getAvailableAudioDevices().then(
(devices: MediaDeviceInfo[]) => {
micDevices = devices;
devices.forEach((device) => {
if (device.deviceId) {
temp_devices.push(device);
}
});
micDevices = temp_devices;
}
);
};
$: try {
update_devices();
} catch (err) {
if (err instanceof DOMException && err.name == "NotAllowedError") {
dispatch("error", i18n("audio.allow_recording_access"));
}
throw err;
}
let permissionName = "microphone" as PermissionName;
navigator.permissions.query({ name: permissionName }).then(function (result) {
if (result.state == "denied") {
device_access_denied = true;
}
});
$: record.on("record-start", () => {
record.startMic();
@ -144,51 +104,15 @@
<time class="duration-button duration">{record_time}</time>
{/if}
</div>
<select
class="mic-select"
aria-label="Select input device"
disabled={micDevices.length === 0}
>
{#if device_access_denied}
<option value="">{i18n("audio.allow_recording_access")}</option>
{:else if micDevices.length === 0 || !micDevices}
<option value="">{i18n("audio.no_microphone")}</option>
{:else}
{#each micDevices as micDevice}
<option value={micDevice.deviceId}>{micDevice.label}</option>
{/each}
{/if}
</select>
<DeviceSelect bind:micDevices {i18n} />
</div>
<style>
.mic-select {
height: var(--size-8);
background: var(--block-background-fill);
padding: 0px var(--spacing-xxl);
border-radius: var(--radius-full);
font-size: var(--text-md);
border: 1px solid var(--neutral-400);
margin: var(--size-1) var(--size-1) 0 0;
}
.controls {
display: flex;
align-items: center;
justify-content: space-between;
flex-wrap: wrap;
overflow: hidden;
}
.controls select {
text-overflow: ellipsis;
max-width: var(--size-40);
}
@media (max-width: 375px) {
.controls select {
width: 100%;
}
}
.wrapper {

70
js/audio/streaming/StreamAudio.svelte
View File

@ -4,6 +4,7 @@
import WaveSurfer from "wavesurfer.js";
import RecordPlugin from "wavesurfer.js/dist/plugins/record.js";
import type { WaveformOptions } from "../shared/types";
import DeviceSelect from "../shared/DeviceSelect.svelte";
export let recording = false;
export let paused_recording = false;
@ -20,6 +21,8 @@
let microphoneContainer: HTMLDivElement;
let micDevices: MediaDeviceInfo[] = [];
onMount(() => {
create_mic_waveform();
});
@ -44,37 +47,48 @@
style:display={recording ? "block" : "none"}
/>
{/if}
{#if recording}
<button
class={paused_recording ? "stop-button-paused" : "stop-button"}
on:click={() => {
waveformRecord?.stopMic();
stop();
}}
>
<span class="record-icon">
<span class="pinger" />
<span class="dot" />
</span>
{paused_recording ? i18n("audio.pause") : i18n("audio.stop")}
</button>
{:else}
<button
class="record-button"
on:click={() => {
waveformRecord?.startMic();
record();
}}
>
<span class="record-icon">
<span class="dot" />
</span>
{i18n("audio.record")}
</button>
{/if}
<div class="controls">
{#if recording}
<button
class={paused_recording ? "stop-button-paused" : "stop-button"}
on:click={() => {
waveformRecord?.stopMic();
stop();
}}
>
<span class="record-icon">
<span class="pinger" />
<span class="dot" />
</span>
{paused_recording ? i18n("audio.pause") : i18n("audio.stop")}
</button>
{:else}
<button
class="record-button"
on:click={() => {
waveformRecord?.startMic();
record();
}}
>
<span class="record-icon">
<span class="dot" />
</span>
{i18n("audio.record")}
</button>
{/if}
<DeviceSelect bind:micDevices {i18n} />
</div>
</div>
<style>
.controls {
display: flex;
align-items: center;
justify-content: space-between;
flex-wrap: wrap;
}
.mic-wrap {
display: block;
align-items: center;