Using marked for chatbot markdown parsing (#4150)

This commit is contained in:
Dawood Khan 2023-05-18 08:55:46 -07:00 committed by GitHub
parent 9ece3b43b9
commit 7145327058
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 195 additions and 282 deletions

View File

@ -40,6 +40,7 @@ No changes to highlight.
## Other Changes:
- Change `gr.Chatbot()` markdown parsing to frontend using `marked` library and `prism` by [@dawoodkhan82](https://github.com/dawoodkhan82) in [PR 4150](https://github.com/gradio-app/gradio/pull/4150)
- Update the js client by [@pngwn](https://github.com/pngwn) in [PR 3899](https://github.com/gradio-app/gradio/pull/3899)
- Fix documentation for the shape of the numpy array produced by the `Image` component by [@der3318](https://github.com/der3318) in [PR 4204](https://github.com/gradio-app/gradio/pull/4204).
- Updates the timeout for websocket messaging from 1 second to 5 seconds by [@abidlabs](https://github.com/abidlabs) in [PR 4235](https://github.com/gradio-app/gradio/pull/4235)

View File

@ -4526,7 +4526,6 @@ class Chatbot(Changeable, Selectable, IOComponent, JSONSerializable):
warnings.warn(
"The 'color_map' parameter has been deprecated.",
)
self.md = utils.get_markdown_parser()
self.select: EventListenerMethod
"""
Event listener for when the user selects message from Chatbot.
@ -4628,9 +4627,6 @@ class Chatbot(Changeable, Selectable, IOComponent, JSONSerializable):
}
elif isinstance(chat_message, str):
chat_message = inspect.cleandoc(chat_message)
chat_message = cast(str, self.md.render(chat_message))
if chat_message.startswith("<p>") and chat_message.endswith("</p>\n"):
chat_message = chat_message[3:-5]
return chat_message
else:
raise ValueError(f"Invalid message for Chatbot component: {chat_message}")

View File

@ -39,9 +39,6 @@ from markdown_it import MarkdownIt
from mdit_py_plugins.dollarmath.index import dollarmath_plugin
from mdit_py_plugins.footnote.index import footnote_plugin
from pydantic import BaseModel, parse_obj_as
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import get_lexer_by_name
import gradio
from gradio.context import Context
@ -890,16 +887,6 @@ def get_serializer_name(block: Block) -> str | None:
return cls.__name__
def highlight_code(code, name, attrs):
try:
lexer = get_lexer_by_name(name)
except Exception:
lexer = get_lexer_by_name("text")
formatter = HtmlFormatter()
return highlight(code, lexer, formatter)
def get_markdown_parser() -> MarkdownIt:
md = (
MarkdownIt(
@ -908,7 +895,6 @@ def get_markdown_parser() -> MarkdownIt:
"linkify": True,
"typographer": True,
"html": True,
"highlight": highlight_code,
},
)
.use(dollarmath_plugin, renderer=tex2svg, allow_digits=False)

View File

@ -3,6 +3,7 @@
import { Block, BlockLabel } from "@gradio/atoms";
import type { LoadingStatus } from "../StatusTracker/types";
import type { Styles } from "@gradio/utils";
import type { ThemeMode } from "js/app/src/components/types";
import { Chat } from "@gradio/icons";
import type { FileData } from "@gradio/upload";
import { normalise_file } from "@gradio/upload";
@ -20,6 +21,7 @@
export let root: string;
export let root_url: null | string;
export let selectable: boolean = false;
export let theme_mode: ThemeMode;
const redirect_src_url = (src: string) =>
src.replace('src="/file', `src="${root}file`);
@ -50,6 +52,7 @@
<ChatBot
{style}
{selectable}
{theme_mode}
value={_value}
pending_message={loading_status?.status === "pending"}
on:change

View File

@ -10,6 +10,11 @@
"dependencies": {
"@gradio/theme": "workspace:^0.0.1",
"@gradio/utils": "workspace:^0.0.1",
"@gradio/upload": "workspace:^0.0.1"
"@gradio/upload": "workspace:^0.0.1",
"@gradio/icons": "workspace:^0.0.1",
"marked": "^5.0.1",
"@types/marked": "^4.3.0",
"prismjs": "1.29.0",
"@types/prismjs": "1.26.0"
}
}

View File

@ -1,8 +1,17 @@
<script lang="ts">
import { marked } from "marked";
import Prism from "prismjs";
import "prismjs/components/prism-python";
import { beforeUpdate, afterUpdate, createEventDispatcher } from "svelte";
import type { Styles, SelectData } from "@gradio/utils";
import type { ThemeMode } from "js/app/src/components/types";
import type { FileData } from "@gradio/upload";
import "./manni.css";
import Copy from "./Copy.svelte";
const code_highlight_css = {
light: () => import("prismjs/themes/prism.css"),
dark: () => import("prismjs/themes/prism-dark.css")
};
export let value: Array<
[string | FileData | null, string | FileData | null]
@ -14,6 +23,32 @@
export let feedback: Array<string> | null = null;
export let style: Styles = {};
export let selectable: boolean = false;
export let theme_mode: ThemeMode;
$: if (theme_mode == "dark") {
code_highlight_css.dark();
} else {
code_highlight_css.light();
}
marked.setOptions({
renderer: new marked.Renderer(),
gfm: true,
breaks: true,
pedantic: false,
sanitize: true,
smartLists: true,
smartypants: false
});
marked.setOptions({
highlight: (code: string, lang: string) => {
if (Prism.languages[lang]) {
return Prism.highlight(code, Prism.languages[lang], lang);
} else {
return code;
}
}
});
let div: HTMLDivElement;
let autoscroll: Boolean;
@ -39,28 +74,22 @@
}
div.querySelectorAll("pre > code").forEach((n) => {
let code_node = n as HTMLElement;
let node = n.parentElement as HTMLElement;
node.style.position = "relative";
const button = document.createElement("button");
button.className = "copy-button";
button.innerHTML = "Copy";
button.style.position = "absolute";
button.style.right = "0";
button.style.top = "0";
button.style.zIndex = "1";
button.style.padding = "var(--spacing-md)";
button.style.marginTop = "12px";
button.style.fontSize = "var(--text-sm)";
button.style.borderBottomLeftRadius = "var(--radius-sm)";
button.style.backgroundColor = "var(--block-label-background-fill)";
button.addEventListener("click", () => {
navigator.clipboard.writeText(code_node.innerText.trimEnd());
button.innerHTML = "Copied!";
setTimeout(() => {
button.innerHTML = "Copy";
}, 1000);
const copy_div = document.createElement("div");
new Copy({
target: copy_div,
props: {
value: code_node.innerText.trimEnd()
}
});
node.appendChild(button);
let node = n.parentElement as HTMLElement;
copy_div.style.position = "absolute";
copy_div.style.right = "0";
copy_div.style.top = "0";
copy_div.style.zIndex = "1";
copy_div.style.padding = "var(--spacing-md)";
copy_div.style.borderBottomLeftRadius = "var(--radius-sm)";
node.style.position = "relative";
node.appendChild(copy_div);
});
});
@ -96,7 +125,7 @@
})}
>
{#if typeof message === "string"}
{@html message}
{@html marked.parse(message)}
{#if feedback && j == 1}
<div class="feedback">
{#each feedback as f}
@ -247,13 +276,6 @@
.dot-flashing:nth-child(3) {
animation-delay: 0.66s;
}
.message-wrap > div :global(.highlight) {
margin-top: var(--spacing-xs);
margin-bottom: var(--spacing-xs);
border-radius: var(--radius-md);
background: var(--chatbot-code-background-color);
padding-left: var(--spacing-xxl);
}
/* Small screen */
@media (max-width: 480px) {
@ -290,7 +312,45 @@
display: none;
}
/* Code blocks */
.message-wrap :global(pre[class*="language-"]),
.message-wrap :global(pre) {
padding: var(--spacing-xl) 0px;
margin-top: var(--spacing-sm);
margin-bottom: var(--spacing-sm);
box-shadow: none;
border: none;
border-radius: var(--radius-md);
background-color: var(--chatbot-code-background-color);
padding: var(--spacing-xl) 10px;
}
/* Tables */
.message-wrap :global(table),
.message-wrap :global(tr),
.message-wrap :global(td),
.message-wrap :global(th) {
margin-top: var(--spacing-sm);
margin-bottom: var(--spacing-sm);
padding: var(--spacing-xl);
}
.message-wrap .bot :global(table),
.message-wrap .bot :global(tr),
.message-wrap .bot :global(td),
.message-wrap .bot :global(th) {
border: 1px solid var(--border-color-primary);
}
.message-wrap .user :global(table),
.message-wrap .user :global(tr),
.message-wrap .user :global(td),
.message-wrap .user :global(th) {
border: 1px solid var(--border-color-accent);
}
/* Lists */
.message-wrap :global(ol),
.message-wrap :global(ul) {
padding-inline-start: 2em;
}
</style>

View File

@ -0,0 +1,59 @@
<script lang="ts">
import { onDestroy } from "svelte";
import { fade } from "svelte/transition";
import { Copy, Check } from "@gradio/icons";
let copied = false;
export let value: string;
let timer: NodeJS.Timeout;
function copy_feedback() {
copied = true;
if (timer) clearTimeout(timer);
timer = setTimeout(() => {
copied = false;
}, 2000);
}
async function handle_copy() {
if ("clipboard" in navigator) {
await navigator.clipboard.writeText(value);
copy_feedback();
}
}
onDestroy(() => {
if (timer) clearTimeout(timer);
});
</script>
<button on:click={handle_copy} title="copy">
<!-- {#if !copied} -->
<span class="copy-text" class:copied><Copy /> </span>
<!-- {/if} -->
{#if copied}
<span class="check" transition:fade><Check /></span>
{/if}
</button>
<style>
button {
position: relative;
cursor: pointer;
padding: 5px;
width: 22px;
height: 22px;
}
.check {
position: absolute;
top: 0;
right: 0;
z-index: var(--layer-top);
background: var(--background-fill-primary);
padding: var(--size-1);
width: 100%;
height: 100%;
color: var(--body-text-color);
}
</style>

View File

@ -1,231 +0,0 @@
.highlight .hll {
background-color: #ffffcc;
}
.highlight .c {
color: #0099ff;
font-style: italic;
} /* Comment */
.highlight .err {
background-color: #ffaaaa;
color: #aa0000;
} /* Error */
.highlight .k {
color: #006699;
font-weight: bold;
} /* Keyword */
.highlight .o {
color: #555555;
} /* Operator */
.highlight .ch {
color: #0099ff;
font-style: italic;
} /* Comment.Hashbang */
.highlight .cm {
color: #0099ff;
font-style: italic;
} /* Comment.Multiline */
.highlight .cp {
color: #009999;
} /* Comment.Preproc */
.highlight .cpf {
color: #0099ff;
font-style: italic;
} /* Comment.PreprocFile */
.highlight .c1 {
color: #0099ff;
font-style: italic;
} /* Comment.Single */
.highlight .cs {
color: #0099ff;
font-style: italic;
font-weight: bold;
} /* Comment.Special */
.highlight .gd {
border: 1px solid #cc0000;
background-color: #ffcccc;
} /* Generic.Deleted */
.highlight .ge {
font-style: italic;
} /* Generic.Emph */
.highlight .gr {
color: #ff0000;
} /* Generic.Error */
.highlight .gh {
color: #003300;
font-weight: bold;
} /* Generic.Heading */
.highlight .gi {
border: 1px solid #00cc00;
background-color: #ccffcc;
} /* Generic.Inserted */
.highlight .go {
color: #aaaaaa;
} /* Generic.Output */
.highlight .gp {
color: #000099;
font-weight: bold;
} /* Generic.Prompt */
.highlight .gs {
font-weight: bold;
} /* Generic.Strong */
.highlight .gu {
color: #003300;
font-weight: bold;
} /* Generic.Subheading */
.highlight .gt {
color: #99cc66;
} /* Generic.Traceback */
.highlight .kc {
color: #006699;
font-weight: bold;
} /* Keyword.Constant */
.highlight .kd {
color: #006699;
font-weight: bold;
} /* Keyword.Declaration */
.highlight .kn {
color: #006699;
font-weight: bold;
} /* Keyword.Namespace */
.highlight .kp {
color: #006699;
} /* Keyword.Pseudo */
.highlight .kr {
color: #006699;
font-weight: bold;
} /* Keyword.Reserved */
.highlight .kt {
color: #007788;
font-weight: bold;
} /* Keyword.Type */
.highlight .m {
color: #ff6600;
} /* Literal.Number */
.highlight .s {
color: #cc3300;
} /* Literal.String */
.highlight .na {
color: #330099;
} /* Name.Attribute */
.highlight .nb {
color: #336666;
} /* Name.Builtin */
.highlight .nc {
color: #00aa88;
font-weight: bold;
} /* Name.Class */
.highlight .no {
color: #336600;
} /* Name.Constant */
.highlight .nd {
color: #9999ff;
} /* Name.Decorator */
.highlight .ni {
color: #999999;
font-weight: bold;
} /* Name.Entity */
.highlight .ne {
color: #cc0000;
font-weight: bold;
} /* Name.Exception */
.highlight .nf {
color: #cc00ff;
} /* Name.Function */
.highlight .nl {
color: #9999ff;
} /* Name.Label */
.highlight .nn {
color: #00ccff;
font-weight: bold;
} /* Name.Namespace */
.highlight .nt {
color: #330099;
font-weight: bold;
} /* Name.Tag */
.highlight .nv {
color: #003333;
} /* Name.Variable */
.highlight .ow {
color: #000000;
font-weight: bold;
} /* Operator.Word */
.highlight .w {
color: #bbbbbb;
} /* Text.Whitespace */
.highlight .mb {
color: #ff6600;
} /* Literal.Number.Bin */
.highlight .mf {
color: #ff6600;
} /* Literal.Number.Float */
.highlight .mh {
color: #ff6600;
} /* Literal.Number.Hex */
.highlight .mi {
color: #ff6600;
} /* Literal.Number.Integer */
.highlight .mo {
color: #ff6600;
} /* Literal.Number.Oct */
.highlight .sa {
color: #cc3300;
} /* Literal.String.Affix */
.highlight .sb {
color: #cc3300;
} /* Literal.String.Backtick */
.highlight .sc {
color: #cc3300;
} /* Literal.String.Char */
.highlight .dl {
color: #cc3300;
} /* Literal.String.Delimiter */
.highlight .sd {
color: #cc3300;
font-style: italic;
} /* Literal.String.Doc */
.highlight .s2 {
color: #cc3300;
} /* Literal.String.Double */
.highlight .se {
color: #cc3300;
font-weight: bold;
} /* Literal.String.Escape */
.highlight .sh {
color: #cc3300;
} /* Literal.String.Heredoc */
.highlight .si {
color: #aa0000;
} /* Literal.String.Interpol */
.highlight .sx {
color: #cc3300;
} /* Literal.String.Other */
.highlight .sr {
color: #33aaaa;
} /* Literal.String.Regex */
.highlight .s1 {
color: #cc3300;
} /* Literal.String.Single */
.highlight .ss {
color: #ffcc33;
} /* Literal.String.Symbol */
.highlight .bp {
color: #336666;
} /* Name.Builtin.Pseudo */
.highlight .fm {
color: #cc00ff;
} /* Name.Function.Magic */
.highlight .vc {
color: #003333;
} /* Name.Variable.Class */
.highlight .vg {
color: #003333;
} /* Name.Variable.Global */
.highlight .vi {
color: #003333;
} /* Name.Variable.Instance */
.highlight .vm {
color: #003333;
} /* Name.Variable.Magic */
.highlight .il {
color: #ff6600;
} /* Literal.Number.Integer.Long */

34
pnpm-lock.yaml generated
View File

@ -358,6 +358,9 @@ importers:
js/chatbot:
dependencies:
'@gradio/icons':
specifier: workspace:^0.0.1
version: link:../icons
'@gradio/theme':
specifier: workspace:^0.0.1
version: link:../theme
@ -367,6 +370,18 @@ importers:
'@gradio/utils':
specifier: workspace:^0.0.1
version: link:../utils
'@types/marked':
specifier: ^4.3.0
version: 4.3.0
'@types/prismjs':
specifier: 1.26.0
version: 1.26.0
marked:
specifier: ^5.0.1
version: 5.0.1
prismjs:
specifier: 1.29.0
version: 1.29.0
js/code:
dependencies:
@ -1905,6 +1920,10 @@ packages:
resolution: {integrity: sha512-qC4bCqYGy1y/NP7dDVr7KJarn+PbX1nSpwA7JXdu0HxT3QYjO8MJ+cntENtHFVy2dRAyBV23OZ6MxsW1AM1L8g==}
dev: false
/@types/marked@4.3.0:
resolution: {integrity: sha512-zK4gSFMjgslsv5Lyvr3O1yCjgmnE4pr8jbG8qVn4QglMwtpvPCf4YT2Wma7Nk95OxUUJI8Z+kzdXohbM7mVpGw==}
dev: false
/@types/minimist@1.2.2:
resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==}
dev: false
@ -1932,6 +1951,10 @@ packages:
resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==}
dev: false
/@types/prismjs@1.26.0:
resolution: {integrity: sha512-ZTaqn/qSqUuAq1YwvOFQfVW1AR/oQJlLSZVustdjwI+GZ8kr0MSHBj0tsXPW1EqHubx50gtBEjbPGsdZwQwCjQ==}
dev: false
/@types/pug@2.0.6:
resolution: {integrity: sha512-SnHmG9wN1UVmagJOnyo/qkk0Z7gejYxOYYmaAwr5u2yFYfsupN3sg10kyzN8Hep/2zbHxCnsumxOoRIRMBwKCg==}
@ -4464,6 +4487,12 @@ packages:
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
dev: false
/marked@5.0.1:
resolution: {integrity: sha512-Nn9peC4lvIZdcfp8Uze6xk4ZYowkcj/K6+e/6rLHadhtjqeip/bYRxMgt3124IGGJ3RPs2uX5YVmAGbUutY18g==}
engines: {node: '>= 18'}
hasBin: true
dev: false
/media-encoder-host-broker@7.0.70:
resolution: {integrity: sha512-ixixE9auojgUHEIQHYvJ75vPxetkHreIfxK20SQ4ZoZSO/vRj4+up72rETMbj2e0UO7xnDJqADsx+sfkoV2eVA==}
dependencies:
@ -5314,6 +5343,11 @@ packages:
react-is: 18.2.0
dev: false
/prismjs@1.29.0:
resolution: {integrity: sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==}
engines: {node: '>=6'}
dev: false
/process-nextick-args@2.0.1:
resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==}
dev: false

View File

@ -1784,7 +1784,7 @@ class TestChatbot:
"""
chatbot = gr.Chatbot()
assert chatbot.postprocess([["You are **cool**\nand fun", "so are *you*"]]) == [
["You are <strong>cool</strong>\nand fun", "so are <em>you</em>"]
["You are **cool**\nand fun", "so are *you*"]
]
multimodal_msg = [