Model3D Gaussian Splatting (#7406)

* add gsplat dependency

add gsplat implementation

run formatting

move example .splat and .ply to url

add changeset

dispose of renderer on unmount

notebook

* dynamically load canvas components

.changeset/tough-baboons-greet.md

@@ -0,0 +1,6 @@
+---
+"@gradio/model3d": minor
+"gradio": minor
+---
+
+feat:Model3D Gaussian Splatting

demo/model3D/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: model3D"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/Bunny.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Bunny.obj\n", "!wget -q -O files/Duck.glb https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Duck.glb\n", "!wget -q -O files/Fox.gltf https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Fox.gltf\n", "!wget -q -O files/face.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/face.obj\n", "!wget -q -O files/sofia.stl https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/sofia.stl\n", "!wget -q -O files/source.txt https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/source.txt"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import os\n", "\n", "\n", "def load_mesh(mesh_file_name):\n", "    return mesh_file_name\n", "\n", "\n", "demo = gr.Interface(\n", "    fn=load_mesh,\n", "    inputs=gr.Model3D(),\n", "    outputs=gr.Model3D(\n", "            clear_color=[0.0, 0.0, 0.0, 0.0],  label=\"3D Model\"),\n", "    examples=[\n", "        [os.path.join(os.path.abspath(''), \"files/Bunny.obj\")],\n", "        [os.path.join(os.path.abspath(''), \"files/Duck.glb\")],\n", "        [os.path.join(os.path.abspath(''), \"files/Fox.gltf\")],\n", "        [os.path.join(os.path.abspath(''), \"files/face.obj\")],\n", "        [os.path.join(os.path.abspath(''), \"files/sofia.stl\")],\n", "    ],\n", "    cache_examples=True\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: model3D"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "os.mkdir('files')\n", "!wget -q -O files/Bunny.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Bunny.obj\n", "!wget -q -O files/Duck.glb https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Duck.glb\n", "!wget -q -O files/Fox.gltf https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/Fox.gltf\n", "!wget -q -O files/face.obj https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/face.obj\n", "!wget -q -O files/sofia.stl https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/sofia.stl\n", "!wget -q -O files/source.txt https://github.com/gradio-app/gradio/raw/main/demo/model3D/files/source.txt"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import os\n", "\n", "\n", "def load_mesh(mesh_file_name):\n", "    return mesh_file_name\n", "\n", "\n", "demo = gr.Interface(\n", "    fn=load_mesh,\n", "    inputs=gr.Model3D(),\n", "    outputs=gr.Model3D(\n", "            clear_color=[0.0, 0.0, 0.0, 0.0],  label=\"3D Model\"),\n", "    examples=[\n", "        [os.path.join(os.path.abspath(''), \"files/Bunny.obj\")],\n", "        [os.path.join(os.path.abspath(''), \"files/Duck.glb\")],\n", "        [os.path.join(os.path.abspath(''), \"files/Fox.gltf\")],\n", "        [os.path.join(os.path.abspath(''), \"files/face.obj\")],\n", "        [os.path.join(os.path.abspath(''), \"files/sofia.stl\")],\n", "        [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bonsai/bonsai-7k-mini.splat\"],\n", "        [\"https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/luigi/luigi.ply\"],\n", "    ],\n", "    cache_examples=True\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/model3D/run.py

@@ -17,6 +17,8 @@ demo = gr.Interface(
         [os.path.join(os.path.dirname(__file__), "files/Fox.gltf")],
         [os.path.join(os.path.dirname(__file__), "files/face.obj")],
         [os.path.join(os.path.dirname(__file__), "files/sofia.stl")],
+        ["https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bonsai/bonsai-7k-mini.splat"],
+        ["https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/luigi/luigi.ply"],
     ],
     cache_examples=True
 )

gradio/components/model3d.py

@@ -15,7 +15,7 @@ from gradio.events import Events
 @document()
 class Model3D(Component):
     """
-    Creates a component allows users to upload or view 3D Model files (.obj, .glb, .stl, or .gltf).
+    Creates a component allows users to upload or view 3D Model files (.obj, .glb, .stl, .gltf, .splat, or .ply).
 
     Demos: model3D
     Guides: how-to-use-3D-model-component
@@ -54,7 +54,7 @@ class Model3D(Component):
     ):
         """
         Parameters:
-            value: path to (.obj, .glb, .stl, or .gltf) file to show in model3D viewer. If callable, the function will be called whenever the app loads to set the initial value of the component.
+            value: path to (.obj, .glb, .stl, .gltf, .splat, or .ply) file to show in model3D viewer. If callable, the function will be called whenever the app loads to set the initial value of the component.
             clear_color: background color of scene, should be a tuple of 4 floats between 0 and 1 representing RGBA values.
             camera_position: initial camera position of scene, provided as a tuple of `(alpha, beta, radius)`. Each value is optional. If provided, `alpha` and `beta` should be in degrees reflecting the angular position along the longitudinal and latitudinal axes, respectively. Radius corresponds to the distance from the center of the object to the camera.
             zoom_speed: the speed of zooming in and out of the scene when the cursor wheel is rotated or when screen is pinched on a mobile device. Should be a positive float, increase this value to make zooming faster, decrease to make it slower. Affects the wheelPrecision property of the camera.

js/model3D/package.json

@@ -17,7 +17,8 @@
 		"@types/babylon": "^6.16.6",
 		"babylonjs": "^4.2.1",
 		"babylonjs-loaders": "^4.2.1",
-		"dequal": "^2.0.2"
+		"dequal": "^2.0.2",
+		"gsplat": "^1.0.5"
 	},
 	"main_changeset": true,
 	"main": "./Index.svelte",

js/model3D/shared/Canvas3DGS.svelte

@@ -0,0 +1,130 @@
+<script lang="ts">
+	import { onMount } from "svelte";
+	import * as SPLAT from "gsplat";
+	import type { FileData } from "@gradio/client";
+	import { resolve_wasm_src } from "@gradio/wasm/svelte";
+
+	export let value: FileData;
+	export let zoom_speed: number;
+	export let pan_speed: number;
+
+	$: url = value.url;
+
+	/* URL resolution for the Wasm mode. */
+	export let resolved_url: typeof url = undefined; // Exposed to be bound to the download link in the parent component.
+	// The prop can be updated before the Promise from `resolve_wasm_src` is resolved.
+	// In such a case, the resolved url for the old `url` has to be discarded,
+	// This variable `latest_url` is used to pick up only the value resolved for the latest `url`.
+	let latest_url: typeof url;
+	$: {
+		// In normal (non-Wasm) Gradio, the original `url` should be used immediately
+		// without waiting for `resolve_wasm_src()` to resolve.
+		// If it waits, a blank element is displayed until the async task finishes
+		// and it leads to undesirable flickering.
+		// So set `resolved_url` immediately above, and update it with the resolved values below later.
+		resolved_url = url;
+
+		if (url) {
+			latest_url = url;
+			const resolving_url = url;
+			resolve_wasm_src(url).then((resolved) => {
+				if (latest_url === resolving_url) {
+					resolved_url = resolved ?? undefined;
+				} else {
+					resolved && URL.revokeObjectURL(resolved);
+				}
+			});
+		}
+	}
+
+	let canvas: HTMLCanvasElement;
+	let scene: SPLAT.Scene;
+	let camera: SPLAT.Camera;
+	let renderer: SPLAT.WebGLRenderer | null = null;
+	let controls: SPLAT.OrbitControls;
+	let mounted = false;
+	let frameId: number | null = null;
+
+	function reset_scene(): void {
+		if (frameId !== null) {
+			cancelAnimationFrame(frameId);
+			frameId = null;
+		}
+
+		if (renderer !== null) {
+			renderer.dispose();
+			renderer = null;
+		}
+
+		scene = new SPLAT.Scene();
+		camera = new SPLAT.Camera();
+		renderer = new SPLAT.WebGLRenderer(canvas);
+		controls = new SPLAT.OrbitControls(camera, canvas);
+		controls.zoomSpeed = zoom_speed;
+		controls.panSpeed = pan_speed;
+
+		if (!value) {
+			return;
+		}
+
+		let loading = false;
+		const load = async (): Promise<void> => {
+			if (loading) {
+				console.error("Already loading");
+				return;
+			}
+			if (!resolved_url) {
+				throw new Error("No resolved URL");
+			}
+			loading = true;
+			if (resolved_url.endsWith(".ply")) {
+				await SPLAT.PLYLoader.LoadAsync(resolved_url, scene, undefined);
+			} else if (resolved_url.endsWith(".splat")) {
+				await SPLAT.Loader.LoadAsync(resolved_url, scene, undefined);
+			} else {
+				throw new Error("Unsupported file type");
+			}
+			loading = false;
+		};
+
+		const frame = (): void => {
+			if (!renderer) {
+				return;
+			}
+
+			if (loading) {
+				frameId = requestAnimationFrame(frame);
+				return;
+			}
+
+			controls.update();
+			renderer.render(scene, camera);
+
+			frameId = requestAnimationFrame(frame);
+		};
+
+		load();
+		frameId = requestAnimationFrame(frame);
+	}
+
+	onMount(() => {
+		if (value != null) {
+			reset_scene();
+		}
+		mounted = true;
+
+		return () => {
+			if (renderer) {
+				renderer.dispose();
+			}
+		};
+	});
+
+	$: ({ path } = value || {
+		path: undefined
+	});
+
+	$: canvas && mounted && path && reset_scene();
+</script>
+
+<canvas bind:this={canvas}></canvas>

js/model3D/shared/Model3D.svelte

@@ -2,7 +2,6 @@
 	import type { FileData } from "@gradio/client";
 	import { BlockLabel, IconButton } from "@gradio/atoms";
 	import { File, Download, Undo } from "@gradio/icons";
-	import Canvas3D from "./Canvas3D.svelte";
 	import type { I18nFormatter } from "@gradio/utils";
 	import { dequal } from "dequal";
 
@@ -22,9 +21,21 @@
 
 	let current_settings = { camera_position, zoom_speed, pan_speed };
 
-	let canvas3d: Canvas3D;
+	let canvas3dgs: any;
+	let canvas3d: any;
+	let use_3dgs = false;
 	let resolved_url: string | undefined;
 
+	async function loadCanvas3D(): Promise<any> {
+		const module = await import("./Canvas3D.svelte");
+		return module.default;
+	}
+
+	async function loadCanvas3DGS(): Promise<any> {
+		const module = await import("./Canvas3DGS.svelte");
+		return module.default;
+	}
+
 	function handle_undo(): void {
 		canvas3d.reset_camera_position(camera_position, zoom_speed, pan_speed);
 	}
@@ -39,6 +50,21 @@
 			current_settings = { camera_position, zoom_speed, pan_speed };
 		}
 	}
+
+	$: {
+		if (value) {
+			use_3dgs = value?.path.endsWith(".splat") || value?.path.endsWith(".ply");
+			if (use_3dgs) {
+				loadCanvas3DGS().then((module) => {
+					canvas3dgs = module;
+				});
+			} else {
+				loadCanvas3D().then((module) => {
+					canvas3d = module;
+				});
+			}
+		}
+	}
 </script>
 
 <BlockLabel
@@ -59,15 +85,25 @@
 			</a>
 		</div>
 
-		<Canvas3D
-			bind:this={canvas3d}
-			bind:resolved_url
-			{value}
-			{clear_color}
-			{camera_position}
-			{zoom_speed}
-			{pan_speed}
-		/>
+		{#if use_3dgs}
+			<svelte:component
+				this={canvas3dgs}
+				bind:resolved_url
+				{value}
+				{zoom_speed}
+				{pan_speed}
+			/>
+		{:else}
+			<svelte:component
+				this={canvas3d}
+				bind:resolved_url
+				{value}
+				{clear_color}
+				{camera_position}
+				{zoom_speed}
+				{pan_speed}
+			/>
+		{/if}
 	</div>
 {/if}
 

js/model3D/shared/Model3DUpload.svelte

@@ -5,7 +5,6 @@
 	import { BlockLabel } from "@gradio/atoms";
 	import { File } from "@gradio/icons";
 	import type { I18nFormatter } from "@gradio/utils";
-	import Canvas3D from "./Canvas3D.svelte";
 
 	export let value: null | FileData;
 	export let clear_color: [number, number, number, number] = [0, 0, 0, 0];
@@ -39,9 +38,22 @@
 		dispatch("change");
 	}
 
-	let canvas3D: Canvas3D;
+	let canvas3d: any;
+	let canvas3dgs: any;
+	let use_3dgs = false;
+
+	async function loadCanvas3D(): Promise<any> {
+		const module = await import("./Canvas3D.svelte");
+		return module.default;
+	}
+
+	async function loadCanvas3DGS(): Promise<any> {
+		const module = await import("./Canvas3DGS.svelte");
+		return module.default;
+	}
+
 	async function handle_undo(): Promise<void> {
-		canvas3D.reset_camera_position(camera_position, zoom_speed, pan_speed);
+		canvas3d.reset_camera_position(camera_position, zoom_speed, pan_speed);
 	}
 
 	const dispatch = createEventDispatcher<{
@@ -54,6 +66,21 @@
 	let dragging = false;
 
 	$: dispatch("drag", dragging);
+
+	$: {
+		if (value) {
+			use_3dgs = value?.path.endsWith(".splat") || value?.path.endsWith(".ply");
+			if (use_3dgs) {
+				loadCanvas3DGS().then((module) => {
+					canvas3dgs = module;
+				});
+			} else {
+				loadCanvas3D().then((module) => {
+					canvas3d = module;
+				});
+			}
+		}
+	}
 </script>
 
 <BlockLabel {show_label} Icon={File} label={label || "3D Model"} />
@@ -62,7 +89,7 @@
 	<Upload
 		on:load={handle_upload}
 		{root}
-		filetype={[".stl", ".obj", ".gltf", ".glb", "model/obj"]}
+		filetype={[".stl", ".obj", ".gltf", ".glb", "model/obj", ".splat", ".ply"]}
 		bind:dragging
 	>
 		<slot />
@@ -76,14 +103,19 @@
 			on:undo={handle_undo}
 			absolute
 		/>
-		<Canvas3D
-			bind:this={canvas3D}
-			{value}
-			{clear_color}
-			{camera_position}
-			{zoom_speed}
-			{pan_speed}
-		></Canvas3D>
+
+		{#if use_3dgs}
+			<svelte:component this={canvas3dgs} {value} {zoom_speed} {pan_speed} />
+		{:else}
+			<svelte:component
+				this={canvas3d}
+				{value}
+				{clear_color}
+				{camera_position}
+				{zoom_speed}
+				{pan_speed}
+			/>
+		{/if}
 	</div>
 {/if}
 

pnpm-lock.yaml

@@ -1182,6 +1182,9 @@ importers:
       dequal:
         specifier: ^2.0.2
         version: 2.0.3
+      gsplat:
+        specifier: ^1.0.5
+        version: 1.0.5
 
   js/number:
     dependencies:
@@ -10985,6 +10988,10 @@ packages:
     engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0}
     dev: false
 
+  /gsplat@1.0.5:
+    resolution: {integrity: sha512-SM85+qMA/UwdDk2lFRmkJvLhD5A+qloRzINmPzjvXJ6Iugl3yAEr8TomVQmge0z1i98bHEoSZrJC/UOKVXq3Hg==}
+    dev: false
+
   /gunzip-maybe@1.4.2:
     resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==}
     hasBin: true