mirror/gradio
2
0
Fork 0
mirror of https://github.com/gradio-app/gradio.git synced 2025-04-06 12:30:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

Model3D point cloud and wireframe display modes (#8687)

Browse Source 
* display modes

* add changeset

* test fixes

* lint

* Update gradio/components/model3d.py

Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

* solid

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Co-authored-by: Abubakar Abid <abubakar@huggingface.co>
This commit is contained in:
Dawood Khan 2024-07-04 14:06:41 -07:00 committed by GitHub
parent 8d7309e0c5
commit bc1d45d874
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 98 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.changeset/wise-animals-go.md Normal file
View File

@ -0,0 +1,6 @@
---
"@gradio/model3d": minor
"gradio": minor
---
feat:Model3D point cloud and wireframe display modes

2
demo/model3D/run.ipynb
View File

@ -1 +1 @@
{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: model3D"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/Bunny.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Bunny.obj\n", "!wget -q -O files/Duck.glb https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Duck.glb\n", "!wget -q -O files/Fox.gltf https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Fox.gltf\n", "!wget -q -O files/face.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/face.obj\n", "!wget -q -O files/sofia.stl https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/sofia.stl\n", "!wget -q -O files/source.txt https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/source.txt"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import os\n", "\n", "\n", "def load_mesh(mesh_file_name):\n", " return mesh_file_name\n", "\n", "\n", "demo = gr.Interface(\n", " fn=load_mesh,\n", " inputs=gr.Model3D(),\n", " outputs=gr.Model3D(\n", " clear_color=[0.0, 0.0, 0.0, 0.0], label=\"3D Model\"),\n", " examples=[\n", " [os.path.join(os.path.abspath(''), \"files/Bunny.obj\")],\n", " [os.path.join(os.path.abspath(''), \"files/Duck.glb\")],\n", " [os.path.join(os.path.abspath(''), \"files/Fox.gltf\")],\n", " [os.path.join(os.path.abspath(''), \"files/face.obj\")],\n", " [os.path.join(os.path.abspath(''), \"files/sofia.stl\")],\n", " [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bonsai/bonsai-7k-mini.splat\"],\n", " [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/luigi/luigi.ply\"],\n", " ],\n", " cache_examples=True\n", ")\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: model3D"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/Bunny.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Bunny.obj\n", "!wget -q -O files/Duck.glb https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Duck.glb\n", "!wget -q -O files/Fox.gltf https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Fox.gltf\n", "!wget -q -O files/face.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/face.obj\n", "!wget -q -O files/sofia.stl https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/sofia.stl\n", "!wget -q -O files/source.txt https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/source.txt"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import os\n", "\n", "\n", "def load_mesh(mesh_file_name):\n", " return mesh_file_name\n", "\n", "demo = gr.Interface(\n", " fn=load_mesh,\n", " inputs=gr.Model3D(),\n", " outputs=gr.Model3D(\n", " clear_color=[0.0, 0.0, 0.0, 0.0], label=\"3D Model\", display_mode=\"wireframe\"),\n", " examples=[\n", " [os.path.join(os.path.abspath(''), \"files/Bunny.obj\")],\n", " [os.path.join(os.path.abspath(''), \"files/Duck.glb\")],\n", " [os.path.join(os.path.abspath(''), \"files/Fox.gltf\")],\n", " [os.path.join(os.path.abspath(''), \"files/face.obj\")],\n", " [os.path.join(os.path.abspath(''), \"files/sofia.stl\")],\n", " [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bonsai/bonsai-7k-mini.splat\"],\n", " [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/luigi/luigi.ply\"],\n", " ],\n", " cache_examples=True\n", ")\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

3
demo/model3D/run.py
View File

@ -5,12 +5,11 @@ import os
def load_mesh(mesh_file_name):
return mesh_file_name
demo = gr.Interface(
fn=load_mesh,
inputs=gr.Model3D(),
outputs=gr.Model3D(
clear_color=[0.0, 0.0, 0.0, 0.0], label="3D Model"),
clear_color=[0.0, 0.0, 0.0, 0.0], label="3D Model", display_mode="wireframe"),
examples=[
[os.path.join(os.path.dirname(__file__), "files/Bunny.obj")],
[os.path.join(os.path.dirname(__file__), "files/Duck.glb")],

5
gradio/components/model3d.py
View File

@ -3,7 +3,7 @@
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING, Callable
from typing import TYPE_CHECKING, Callable, Literal
from gradio_client import handle_file
from gradio_client.documentation import document
@ -32,6 +32,7 @@ class Model3D(Component):
self,
value: str | Callable | None = None,
*,
display_mode: Literal["solid", "point_cloud", "wireframe"] | None = None,
clear_color: tuple[float, float, float, float] | None = None,
camera_position: tuple[
int | float | None, int | float | None, int | float | None
@ -60,6 +61,7 @@ class Model3D(Component):
"""
Parameters:
value: path to (.obj, .glb, .stl, .gltf, .splat, or .ply) file to show in model3D viewer. If callable, the function will be called whenever the app loads to set the initial value of the component.
display_mode: the display mode of the 3D model in the scene. Can be "solid" (which renders the model as a solid object), "point_cloud", or "wireframe". For .splat, or .ply files, this parameter is ignored, as those files can only be rendered as solid objects.
clear_color: background color of scene, should be a tuple of 4 floats between 0 and 1 representing RGBA values.
camera_position: initial camera position of scene, provided as a tuple of `(alpha, beta, radius)`. Each value is optional. If provided, `alpha` and `beta` should be in degrees reflecting the angular position along the longitudinal and latitudinal axes, respectively. Radius corresponds to the distance from the center of the object to the camera.
zoom_speed: the speed of zooming in and out of the scene when the cursor wheel is rotated or when screen is pinched on a mobile device. Should be a positive float, increase this value to make zooming faster, decrease to make it slower. Affects the wheelPrecision property of the camera.
@ -79,6 +81,7 @@ class Model3D(Component):
render: If False, component will not render be rendered in the Blocks context. Should be used if the intention is to assign event listeners now but render the component later.
key: if assigned, will be used to assume identity across a re-render. Components that have the same key across a re-render will have their value preserved.
"""
self.display_mode = display_mode
self.clear_color = clear_color or [0, 0, 0, 0]
self.camera_position = camera_position
self.height = height

3
js/model3D/Index.svelte
View File

@ -20,6 +20,7 @@
export let visible = true;
export let value: null | FileData = null;
export let root: string;
export let display_mode: "solid" | "point_cloud" | "wireframe" = "solid";
export let clear_color: [number, number, number, number];
export let loading_status: LoadingStatus;
export let label: string;
@ -66,6 +67,7 @@
<Model3D
{value}
i18n={gradio.i18n}
{display_mode}
{clear_color}
{label}
{show_label}
@ -105,6 +107,7 @@
{label}
{show_label}
{root}
{display_mode}
{clear_color}
{value}
{camera_position}

79
js/model3D/shared/Canvas3D.svelte
View File

@ -15,6 +15,7 @@
}
export let value: FileData;
export let display_mode: "solid" | "point_cloud" | "wireframe";
export let clear_color: [number, number, number, number];
export let camera_position: [number | null, number | null, number | null];
export let zoom_speed: number;
@ -53,6 +54,7 @@
let canvas: HTMLCanvasElement;
let scene: BABYLON.Scene;
let engine: BABYLON.Engine;
let point_cloud_system: BABYLON.PointsCloudSystem | null = null;
let mounted = false;
onMount(() => {
@ -91,6 +93,11 @@
mesh.dispose();
});
if (point_cloud_system) {
point_cloud_system.dispose();
point_cloud_system = null;
}
// Load the new model
if (url) {
BABYLON.SceneLoader.ShowLoadingScreen = false;
@ -98,7 +105,15 @@
url,
"",
scene,
() => create_camera(scene, camera_position, zoom_speed, pan_speed),
() => {
if (display_mode === "point_cloud") {
create_point_cloud(scene);
} else if (display_mode === "wireframe") {
create_wireframe(scene);
} else {
create_camera(scene, camera_position, zoom_speed, pan_speed);
}
},
undefined,
undefined,
"." + value.path.split(".").pop()
@ -145,6 +160,68 @@
create_camera(scene, camera_position, zoom_speed, pan_speed);
}
}
function create_point_cloud(scene: BABYLON.Scene): void {
const meshes = scene.meshes;
const pointPositions: BABYLON.Vector3[] = [];
meshes.forEach((mesh) => {
if (mesh instanceof BABYLON.Mesh) {
const positions = mesh.getVerticesData(
BABYLON.VertexBuffer.PositionKind
);
if (positions) {
for (let i = 0; i < positions.length; i += 3) {
pointPositions.push(
new BABYLON.Vector3(
positions[i],
positions[i + 1],
positions[i + 2]
)
);
}
}
mesh.setEnabled(false);
}
});
point_cloud_system = new BABYLON.PointsCloudSystem(
"point_cloud_system",
1,
scene
);
point_cloud_system.addPoints(
pointPositions.length,
(particle: BABYLON.CloudPoint, i: number) => {
particle.position = pointPositions[i];
particle.color = new BABYLON.Color4(
Math.random(),
Math.random(),
Math.random(),
1.0
);
}
);
point_cloud_system.buildMeshAsync().then((mesh) => {
mesh.alwaysSelectAsActiveMesh = true;
create_camera(scene, camera_position, zoom_speed, pan_speed);
});
}
function create_wireframe(scene: BABYLON.Scene): void {
scene.meshes.forEach((mesh) => {
if (mesh instanceof BABYLON.Mesh) {
mesh.material = new BABYLON.StandardMaterial(
"wireframeMaterial",
scene
);
mesh.material.wireframe = true;
}
create_camera(scene, camera_position, zoom_speed, pan_speed);
});
}
</script>
<canvas bind:this={canvas}></canvas>

2
js/model3D/shared/Model3D.svelte
View File

@ -8,6 +8,7 @@
import type Canvas3D from "./Canvas3D.svelte";
export let value: FileData | null;
export let display_mode: "solid" | "point_cloud" | "wireframe" = "solid";
export let clear_color: [number, number, number, number] = [0, 0, 0, 0];
export let label = "";
export let show_label: boolean;
@ -101,6 +102,7 @@
bind:this={canvas3d}
bind:resolved_url
{value}
{display_mode}
{clear_color}
{camera_position}
{zoom_speed}

2
js/model3D/shared/Model3DUpload.svelte
View File

@ -9,6 +9,7 @@
import type Canvas3D from "./Canvas3D.svelte";
export let value: null | FileData;
export let display_mode: "solid" | "point_cloud" | "wireframe" = "solid";
export let clear_color: [number, number, number, number] = [0, 0, 0, 0];
export let label = "";
export let show_label: boolean;
@ -121,6 +122,7 @@
this={Canvas3DComponent}
bind:this={canvas3d}
{value}
{display_mode}
{clear_color}
{camera_position}
{zoom_speed}

1
test/components/test_model3d.py
View File

@ -11,6 +11,7 @@ class TestModel3D:
model_component = gr.components.Model3D(None, label="Model")
assert model_component.get_config() == {
"value": None,
"display_mode": None,
"clear_color": [0, 0, 0, 0],
"label": "Model",
"show_label": True,