mirror of
https://github.com/gradio-app/gradio.git
synced 2025-03-31 12:20:26 +08:00
Allow Chatbot examples to show more than one image (#10111)
* change example image limit to 4 * add changeset * refactor examples * redesign examples * aria improvements * check for multimodal * use multimedia icons * clean up * remove min-height * add param * fix test * Revert "add param" This reverts commit f858097240c3615c19650b5d5263c70d6a408fac. * Revert "fix test" This reverts commit 41570fa1004965479e7692ee83dd33b1d4c709b0. * handle icon in chat_interface.py * remove unused css class * fix types * format * fix test * - change logic to use example icon for text example if exists - ensure mixed files can be displayed in multiples * add stories * tweaks --------- Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com> Co-authored-by: Abubakar Abid <abubakar@huggingface.co>
This commit is contained in:
Hannah
GitHub
parent
20b9d72ebb
commit
3665e81b53
6
.changeset/legal-aliens-burn.md
Normal file
6
.changeset/legal-aliens-burn.md
Normal file
@ -0,0 +1,6 @@
|
||||
---
|
||||
"@gradio/chatbot": minor
|
||||
"gradio": minor
|
||||
---
|
||||
|
||||
feat:Allow Chatbot examples to show more than one image
|
||||
@ -319,8 +319,8 @@ class ChatInterface(Blocks):
|
||||
self.show_progress = show_progress
|
||||
self._setup_events()
|
||||
|
||||
@staticmethod
|
||||
def _setup_example_messages(
|
||||
self,
|
||||
examples: list[str] | list[MultimodalValue] | list[list] | None,
|
||||
example_labels: list[str] | None = None,
|
||||
example_icons: list[str] | None = None,
|
||||
@ -338,8 +338,21 @@ class ChatInterface(Blocks):
|
||||
example_message["files"] = example.get("files", [])
|
||||
if example_labels:
|
||||
example_message["display_text"] = example_labels[index]
|
||||
if example_icons:
|
||||
example_message["icon"] = example_icons[index]
|
||||
if self.multimodal:
|
||||
example_files = example_message.get("files")
|
||||
if not example_files:
|
||||
if example_icons:
|
||||
example_message["icon"] = example_icons[index]
|
||||
else:
|
||||
example_message["icon"] = {
|
||||
"path": "",
|
||||
"url": None,
|
||||
"orig_name": None,
|
||||
"mime_type": "text", # for internal use, not a valid mime type
|
||||
"meta": {"_type": "gradio.FileData"},
|
||||
}
|
||||
elif example_icons:
|
||||
example_message["icon"] = example_icons[index]
|
||||
examples_messages.append(example_message)
|
||||
return examples_messages
|
||||
|
||||
|
||||
@ -223,3 +223,138 @@ This document is a showcase of various Markdown capabilities.`,
|
||||
]
|
||||
}}
|
||||
/>
|
||||
|
||||
<Story
|
||||
name="MultimodalChatbot with examples"
|
||||
args={{
|
||||
value: [],
|
||||
examples: [
|
||||
{
|
||||
text: "What is machine learning?",
|
||||
icon: { mime_type: "text" }
|
||||
},
|
||||
{
|
||||
text: "Analyze this image",
|
||||
files: [
|
||||
{
|
||||
mime_type: "image/jpeg",
|
||||
url: "https://raw.githubusercontent.com/gradio-app/gradio/main/test/test_files/bus.png",
|
||||
orig_name: "bus.png"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
text: "Process this document",
|
||||
files: [
|
||||
{
|
||||
mime_type: "application/pdf",
|
||||
url: "/document.pdf",
|
||||
orig_name: "document.pdf"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
text: "Compare these images",
|
||||
files: [
|
||||
{
|
||||
mime_type: "image/jpeg",
|
||||
url: "https://raw.githubusercontent.com/gradio-app/gradio/main/test/test_files/bus.png",
|
||||
orig_name: "image1.jpg"
|
||||
},
|
||||
{
|
||||
mime_type: "image/jpeg",
|
||||
url: "https://raw.githubusercontent.com/gradio-app/gradio/main/test/test_files/bus.png",
|
||||
orig_name: "image2.jpg"
|
||||
},
|
||||
{
|
||||
mime_type: "image/jpeg",
|
||||
url: "https://raw.githubusercontent.com/gradio-app/gradio/main/test/test_files/bus.png",
|
||||
orig_name: "image3.jpg"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
text: "Process these files",
|
||||
files: [
|
||||
{
|
||||
mime_type: "application/pdf",
|
||||
url: "doc1.pdf",
|
||||
orig_name: "document1.pdf"
|
||||
},
|
||||
{
|
||||
mime_type: "application/pdf",
|
||||
url: "/doc2.pdf",
|
||||
orig_name: "document2.pdf"
|
||||
},
|
||||
{
|
||||
mime_type: "application/pdf",
|
||||
url: "/doc3.pdf",
|
||||
orig_name: "document3.pdf"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
text: "Analyze this dataset",
|
||||
files: [
|
||||
{
|
||||
mime_type: "image/jpeg",
|
||||
url: "https://raw.githubusercontent.com/gradio-app/gradio/main/test/test_files/bus.png",
|
||||
orig_name: "visualization.jpg"
|
||||
},
|
||||
{
|
||||
mime_type: "image/jpeg",
|
||||
url: "https://raw.githubusercontent.com/gradio-app/gradio/main/test/test_files/bus.png",
|
||||
orig_name: "visualization.jpg"
|
||||
},
|
||||
{
|
||||
mime_type: "application/pdf",
|
||||
url: "/data.pdf",
|
||||
orig_name: "data.pdf"
|
||||
},
|
||||
{
|
||||
mime_type: "audio/mp3",
|
||||
url: "/audio.mp3",
|
||||
orig_name: "recording.mp3"
|
||||
},
|
||||
{
|
||||
mime_type: "video/mp4",
|
||||
url: "/video.mp4",
|
||||
orig_name: "video.mp4"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}}
|
||||
/>
|
||||
|
||||
<Story
|
||||
name="Chatbot with examples (not multimodal)"
|
||||
args={{
|
||||
value: [],
|
||||
examples: [
|
||||
{
|
||||
text: "What is machine learning?"
|
||||
},
|
||||
{
|
||||
text: "Analyze this image",
|
||||
files: [
|
||||
{
|
||||
mime_type: "image/jpeg",
|
||||
url: "https://raw.githubusercontent.com/gradio-app/gradio/main/test/test_files/bus.png",
|
||||
orig_name: "bus.png"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
text: "Process this document",
|
||||
files: [
|
||||
{
|
||||
mime_type: "application/pdf",
|
||||
url: "/document.pdf",
|
||||
orig_name: "document.pdf"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}}
|
||||
/>
|
||||
|
||||
@ -34,6 +34,8 @@
|
||||
import { ShareError } from "@gradio/utils";
|
||||
import { Gradio } from "@gradio/utils";
|
||||
|
||||
import Examples from "./Examples.svelte";
|
||||
|
||||
export let value: NormalisedMessage[] | null = [];
|
||||
let old_value: NormalisedMessage[] | null = null;
|
||||
|
||||
@ -328,55 +330,13 @@
|
||||
{/if}
|
||||
</div>
|
||||
{:else}
|
||||
<div class="placeholder-content">
|
||||
{#if placeholder !== null}
|
||||
<div class="placeholder">
|
||||
<Markdown message={placeholder} {latex_delimiters} {root} />
|
||||
</div>
|
||||
{/if}
|
||||
{#if examples !== null}
|
||||
<div class="examples">
|
||||
{#each examples as example, i}
|
||||
<button
|
||||
class="example"
|
||||
on:click={() => handle_example_select(i, example)}
|
||||
>
|
||||
{#if example.icon !== undefined}
|
||||
<div class="example-icon-container">
|
||||
<Image
|
||||
class="example-icon"
|
||||
src={example.icon.url}
|
||||
alt="example-icon"
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
{#if example.display_text !== undefined}
|
||||
<span class="example-display-text">{example.display_text}</span>
|
||||
{:else}
|
||||
<span class="example-text">{example.text}</span>
|
||||
{/if}
|
||||
{#if example.files !== undefined && example.files.length > 1}
|
||||
<span class="example-file"
|
||||
><em>{example.files.length} Files</em></span
|
||||
>
|
||||
{:else if example.files !== undefined && example.files[0] !== undefined && example.files[0].mime_type?.includes("image")}
|
||||
<div class="example-image-container">
|
||||
<Image
|
||||
class="example-image"
|
||||
src={example.files[0].url}
|
||||
alt="example-image"
|
||||
/>
|
||||
</div>
|
||||
{:else if example.files !== undefined && example.files[0] !== undefined}
|
||||
<span class="example-file"
|
||||
><em>{example.files[0].orig_name}</em></span
|
||||
>
|
||||
{/if}
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
<Examples
|
||||
{examples}
|
||||
{placeholder}
|
||||
{latex_delimiters}
|
||||
{root}
|
||||
on:example_select={(e) => dispatch("example_select", e.detail)}
|
||||
/>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
@ -392,93 +352,6 @@
|
||||
{/if}
|
||||
|
||||
<style>
|
||||
.placeholder-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.placeholder {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
height: 100%;
|
||||
flex-grow: 1;
|
||||
}
|
||||
|
||||
.examples :global(img) {
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.examples {
|
||||
margin: auto;
|
||||
padding: var(--spacing-xxl);
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: var(--spacing-xxl);
|
||||
max-width: calc(min(4 * 200px + 5 * var(--spacing-xxl), 100%));
|
||||
}
|
||||
|
||||
.example {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
padding: var(--spacing-xl);
|
||||
border: 0.05px solid var(--border-color-primary);
|
||||
border-radius: var(--radius-md);
|
||||
background-color: var(--background-fill-secondary);
|
||||
cursor: pointer;
|
||||
transition: var(--button-transition);
|
||||
max-width: var(--size-56);
|
||||
width: 100%;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.example:hover {
|
||||
background-color: var(--color-accent-soft);
|
||||
border-color: var(--border-color-accent);
|
||||
}
|
||||
|
||||
.example-icon-container {
|
||||
display: flex;
|
||||
align-self: flex-start;
|
||||
margin-left: var(--spacing-md);
|
||||
width: var(--size-6);
|
||||
height: var(--size-6);
|
||||
}
|
||||
|
||||
.example-display-text,
|
||||
.example-text,
|
||||
.example-file {
|
||||
font-size: var(--text-md);
|
||||
width: 100%;
|
||||
text-align: center;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.example-display-text,
|
||||
.example-file {
|
||||
margin-top: var(--spacing-md);
|
||||
}
|
||||
|
||||
.example-image-container {
|
||||
flex-grow: 1;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
margin-top: var(--spacing-xl);
|
||||
}
|
||||
|
||||
.example-image-container :global(img) {
|
||||
max-height: 100%;
|
||||
max-width: 100%;
|
||||
height: var(--size-32);
|
||||
width: 100%;
|
||||
object-fit: cover;
|
||||
border-radius: var(--radius-xl);
|
||||
}
|
||||
|
||||
.panel-wrap {
|
||||
width: 100%;
|
||||
overflow-y: auto;
|
||||
|
||||
324
js/chatbot/shared/Examples.svelte
Normal file
324
js/chatbot/shared/Examples.svelte
Normal file
@ -0,0 +1,324 @@
|
||||
<script lang="ts">
|
||||
import { Image } from "@gradio/image/shared";
|
||||
import { MarkdownCode as Markdown } from "@gradio/markdown-code";
|
||||
import { File, Music, Video } from "@gradio/icons";
|
||||
import type { ExampleMessage } from "../types";
|
||||
import { createEventDispatcher } from "svelte";
|
||||
import type { SelectData } from "@gradio/utils";
|
||||
import type { FileData } from "@gradio/client";
|
||||
|
||||
export let examples: ExampleMessage[] | null = null;
|
||||
export let placeholder: string | null = null;
|
||||
export let latex_delimiters: {
|
||||
left: string;
|
||||
right: string;
|
||||
display: boolean;
|
||||
}[];
|
||||
export let root: string;
|
||||
|
||||
const dispatch = createEventDispatcher<{
|
||||
example_select: SelectData;
|
||||
}>();
|
||||
|
||||
function handle_example_select(
|
||||
i: number,
|
||||
example: ExampleMessage | string
|
||||
): void {
|
||||
const example_obj =
|
||||
typeof example === "string" ? { text: example } : example;
|
||||
dispatch("example_select", {
|
||||
index: i,
|
||||
value: { text: example_obj.text, files: example_obj.files }
|
||||
});
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="placeholder-content" role="complementary">
|
||||
{#if placeholder !== null}
|
||||
<div class="placeholder">
|
||||
<Markdown message={placeholder} {latex_delimiters} {root} />
|
||||
</div>
|
||||
{/if}
|
||||
{#if examples !== null}
|
||||
<div class="examples" role="list">
|
||||
{#each examples as example, i}
|
||||
<button
|
||||
class="example"
|
||||
on:click={() =>
|
||||
handle_example_select(
|
||||
i,
|
||||
typeof example === "string" ? { text: example } : example
|
||||
)}
|
||||
aria-label={`Select example ${i + 1}: ${example.display_text || example.text}`}
|
||||
>
|
||||
<div class="example-content">
|
||||
{#if example?.icon?.url}
|
||||
<div class="example-image-container">
|
||||
<Image
|
||||
class="example-image"
|
||||
src={example.icon.url}
|
||||
alt="Example icon"
|
||||
/>
|
||||
</div>
|
||||
{:else if example?.icon?.mime_type === "text"}
|
||||
<div class="example-icon" aria-hidden="true">
|
||||
<span class="text-icon-aa">Aa</span>
|
||||
</div>
|
||||
{:else if example.files !== undefined && example.files.length > 0}
|
||||
{#if example.files.length > 1}
|
||||
<div
|
||||
class="example-icons-grid"
|
||||
role="group"
|
||||
aria-label="Example attachments"
|
||||
>
|
||||
{#each example.files.slice(0, 4) as file, i}
|
||||
{#if file.mime_type?.includes("image")}
|
||||
<div class="example-image-container">
|
||||
<Image
|
||||
class="example-image"
|
||||
src={file.url}
|
||||
alt={file.orig_name || `Example image ${i + 1}`}
|
||||
/>
|
||||
{#if i === 3 && example.files.length > 4}
|
||||
<div
|
||||
class="image-overlay"
|
||||
role="status"
|
||||
aria-label={`${example.files.length - 4} more files`}
|
||||
>
|
||||
+{example.files.length - 4}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
{:else if file.mime_type?.includes("video")}
|
||||
<div class="example-image-container">
|
||||
<video
|
||||
class="example-image"
|
||||
src={file.url}
|
||||
aria-hidden="true"
|
||||
/>
|
||||
{#if i === 3 && example.files.length > 4}
|
||||
<div
|
||||
class="image-overlay"
|
||||
role="status"
|
||||
aria-label={`${example.files.length - 4} more files`}
|
||||
>
|
||||
+{example.files.length - 4}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
{:else}
|
||||
<div
|
||||
class="example-icon"
|
||||
aria-label={`File: ${file.orig_name}`}
|
||||
>
|
||||
{#if file.mime_type?.includes("audio")}
|
||||
<Music />
|
||||
{:else}
|
||||
<File />
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
{/each}
|
||||
{#if example.files.length > 4}
|
||||
<div class="example-icon">
|
||||
<div
|
||||
class="file-overlay"
|
||||
role="status"
|
||||
aria-label={`${example.files.length - 4} more files`}
|
||||
>
|
||||
+{example.files.length - 4}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
{:else if example.files[0].mime_type?.includes("image")}
|
||||
<div class="example-image-container">
|
||||
<Image
|
||||
class="example-image"
|
||||
src={example.files[0].url}
|
||||
alt={example.files[0].orig_name || "Example image"}
|
||||
/>
|
||||
</div>
|
||||
{:else if example.files[0].mime_type?.includes("video")}
|
||||
<div class="example-image-container">
|
||||
<video
|
||||
class="example-image"
|
||||
src={example.files[0].url}
|
||||
aria-hidden="true"
|
||||
/>
|
||||
</div>
|
||||
{:else if example.files[0].mime_type?.includes("audio")}
|
||||
<div
|
||||
class="example-icon"
|
||||
aria-label={`File: ${example.files[0].orig_name}`}
|
||||
>
|
||||
<Music />
|
||||
</div>
|
||||
{:else}
|
||||
<div
|
||||
class="example-icon"
|
||||
aria-label={`File: ${example.files[0].orig_name}`}
|
||||
>
|
||||
<File />
|
||||
</div>
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
<div class="example-text-content">
|
||||
<span class="example-text"
|
||||
>{example.display_text || example.text}</span
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.placeholder-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.placeholder {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
height: 100%;
|
||||
flex-grow: 1;
|
||||
}
|
||||
|
||||
.examples :global(img) {
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.examples {
|
||||
margin: auto;
|
||||
padding: var(--spacing-xxl);
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
|
||||
gap: var(--spacing-xl);
|
||||
max-width: calc(min(4 * 240px + 5 * var(--spacing-xxl), 100%));
|
||||
}
|
||||
|
||||
.example {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
padding: var(--spacing-xxl);
|
||||
border: none;
|
||||
border-radius: var(--radius-lg);
|
||||
background-color: var(--block-background-fill);
|
||||
cursor: pointer;
|
||||
transition: all 150ms ease-in-out;
|
||||
width: 100%;
|
||||
gap: var(--spacing-sm);
|
||||
border: var(--block-border-width) solid var(--block-border-color);
|
||||
transform: translateY(0px);
|
||||
}
|
||||
|
||||
.example:hover {
|
||||
transform: translateY(-2px);
|
||||
background-color: var(--color-accent-soft);
|
||||
}
|
||||
|
||||
.example-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.example-text-content {
|
||||
margin-top: auto;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.example-text {
|
||||
font-size: var(--text-md);
|
||||
text-align: left;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.example-icons-grid {
|
||||
display: flex;
|
||||
gap: var(--spacing-sm);
|
||||
margin-bottom: var(--spacing-lg);
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.example-icon {
|
||||
flex-shrink: 0;
|
||||
width: var(--size-8);
|
||||
height: var(--size-8);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
border-radius: var(--radius-lg);
|
||||
border: var(--block-border-width) solid var(--block-border-color);
|
||||
background-color: var(--block-background-fill);
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.example-icon :global(svg) {
|
||||
width: var(--size-4);
|
||||
height: var(--size-4);
|
||||
color: var(--color-text-secondary);
|
||||
}
|
||||
|
||||
.text-icon-aa {
|
||||
font-size: var(--text-sm);
|
||||
font-weight: var(--weight-semibold);
|
||||
color: var(--color-text-secondary);
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.example-image-container {
|
||||
width: var(--size-8);
|
||||
height: var(--size-8);
|
||||
border-radius: var(--radius-lg);
|
||||
overflow: hidden;
|
||||
position: relative;
|
||||
margin-bottom: var(--spacing-lg);
|
||||
}
|
||||
|
||||
.example-image-container :global(img) {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.image-overlay {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
color: white;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: var(--text-lg);
|
||||
font-weight: var(--weight-semibold);
|
||||
border-radius: var(--radius-lg);
|
||||
}
|
||||
|
||||
.file-overlay {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
color: white;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: var(--text-sm);
|
||||
font-weight: var(--weight-semibold);
|
||||
border-radius: var(--radius-lg);
|
||||
}
|
||||
</style>
|
||||
@ -20,7 +20,7 @@ for (const test_case of cases) {
|
||||
|
||||
// Clear the chat and click on a different example, the input/response are correct
|
||||
await page.getByLabel("Clear").click();
|
||||
await page.getByRole("button", { name: "hola example-image" }).click();
|
||||
await page.getByRole("button", { name: "Select example 2: hola" }).click();
|
||||
await expect(page.locator(".user img")).toBeVisible();
|
||||
await expect(page.locator(".user p")).toContainText("hola");
|
||||
await expect(page.locator(".bot p")).toContainText(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user