mirror of
https://github.com/gradio-app/gradio.git
synced 2025-01-30 11:00:11 +08:00
Lite: Wasm-compatible Model3D (#7315)
* Make Model3DUpload.svelte Wasm-compatible * Make Model3D.svelte Wasm-compatible * add changeset * Create <Canvas3D /> and use it from <Model3D /> and <Model3DUpload /> --------- Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com> Co-authored-by: Abubakar Abid <abubakar@huggingface.co>
This commit is contained in:
Yuichiro Tachibana (Tsuchiya)
GitHub
parent
aea14c4496
commit
b3a9c83095
6
.changeset/silly-apples-wait.md
Normal file
6
.changeset/silly-apples-wait.md
Normal file
@ -0,0 +1,6 @@
|
||||
---
|
||||
"@gradio/model3d": minor
|
||||
"gradio": minor
|
||||
---
|
||||
|
||||
feat:Lite: Wasm-compatible Model3D
|
||||
@ -13,6 +13,7 @@
|
||||
"@gradio/statustracker": "workspace:^",
|
||||
"@gradio/upload": "workspace:^",
|
||||
"@gradio/utils": "workspace:^",
|
||||
"@gradio/wasm": "workspace:^",
|
||||
"@types/babylon": "^6.16.6",
|
||||
"babylonjs": "^4.2.1",
|
||||
"babylonjs-loaders": "^4.2.1",
|
||||
|
||||
149
js/model3D/shared/Canvas3D.svelte
Normal file
149
js/model3D/shared/Canvas3D.svelte
Normal file
@ -0,0 +1,149 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from "svelte";
|
||||
import * as BABYLON from "babylonjs";
|
||||
import * as BABYLON_LOADERS from "babylonjs-loaders";
|
||||
import type { FileData } from "@gradio/client";
|
||||
import { resolve_wasm_src } from "@gradio/wasm/svelte";
|
||||
|
||||
$: {
|
||||
if (
|
||||
BABYLON_LOADERS.OBJFileLoader != undefined &&
|
||||
!BABYLON_LOADERS.OBJFileLoader.IMPORT_VERTEX_COLORS
|
||||
) {
|
||||
BABYLON_LOADERS.OBJFileLoader.IMPORT_VERTEX_COLORS = true;
|
||||
}
|
||||
}
|
||||
|
||||
export let value: FileData;
|
||||
export let clear_color: [number, number, number, number];
|
||||
export let camera_position: [number | null, number | null, number | null];
|
||||
export let zoom_speed: number;
|
||||
export let pan_speed: number;
|
||||
|
||||
$: url = value.url;
|
||||
|
||||
/* URL resolution for the Wasm mode. */
|
||||
export let resolved_url: typeof url = undefined; // Exposed to be bound to the download link in the parent component.
|
||||
// The prop can be updated before the Promise from `resolve_wasm_src` is resolved.
|
||||
// In such a case, the resolved url for the old `url` has to be discarded,
|
||||
// This variable `latest_url` is used to pick up only the value resolved for the latest `url`.
|
||||
let latest_url: typeof url;
|
||||
$: {
|
||||
// In normal (non-Wasm) Gradio, the original `url` should be used immediately
|
||||
// without waiting for `resolve_wasm_src()` to resolve.
|
||||
// If it waits, a blank element is displayed until the async task finishes
|
||||
// and it leads to undesirable flickering.
|
||||
// So set `resolved_url` immediately above, and update it with the resolved values below later.
|
||||
resolved_url = url;
|
||||
|
||||
if (url) {
|
||||
latest_url = url;
|
||||
const resolving_url = url;
|
||||
resolve_wasm_src(url).then((resolved) => {
|
||||
if (latest_url === resolving_url) {
|
||||
resolved_url = resolved ?? undefined;
|
||||
} else {
|
||||
resolved && URL.revokeObjectURL(resolved);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/* BabylonJS engine and scene management */
|
||||
let canvas: HTMLCanvasElement;
|
||||
let scene: BABYLON.Scene;
|
||||
let engine: BABYLON.Engine;
|
||||
let mounted = false;
|
||||
|
||||
onMount(() => {
|
||||
// Initialize BabylonJS engine and scene
|
||||
engine = new BABYLON.Engine(canvas, true);
|
||||
scene = new BABYLON.Scene(engine);
|
||||
|
||||
scene.createDefaultCameraOrLight();
|
||||
scene.clearColor = scene.clearColor = new BABYLON.Color4(...clear_color);
|
||||
|
||||
engine.runRenderLoop(() => {
|
||||
scene.render();
|
||||
});
|
||||
|
||||
function onWindowResize(): void {
|
||||
engine.resize();
|
||||
}
|
||||
window.addEventListener("resize", onWindowResize);
|
||||
|
||||
mounted = true;
|
||||
|
||||
return () => {
|
||||
scene.dispose();
|
||||
engine.dispose();
|
||||
window.removeEventListener("resize", onWindowResize);
|
||||
};
|
||||
});
|
||||
|
||||
$: mounted && load_model(resolved_url);
|
||||
|
||||
function load_model(url: string | undefined): void {
|
||||
if (scene) {
|
||||
// Dispose of the previous model before loading a new one
|
||||
scene.meshes.forEach((mesh) => {
|
||||
mesh.dispose();
|
||||
});
|
||||
|
||||
// Load the new model
|
||||
if (url) {
|
||||
BABYLON.SceneLoader.ShowLoadingScreen = false;
|
||||
BABYLON.SceneLoader.Append(
|
||||
url,
|
||||
"",
|
||||
scene,
|
||||
() => create_camera(scene, camera_position, zoom_speed, pan_speed),
|
||||
undefined,
|
||||
undefined,
|
||||
"." + value.path.split(".").pop()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function create_camera(
|
||||
scene: BABYLON.Scene,
|
||||
camera_position: [number | null, number | null, number | null],
|
||||
zoom_speed: number,
|
||||
pan_speed: number
|
||||
): void {
|
||||
scene.createDefaultCamera(true, true, true);
|
||||
var helperCamera = scene.activeCamera! as BABYLON.ArcRotateCamera;
|
||||
if (camera_position[0] !== null) {
|
||||
helperCamera.alpha = BABYLON.Tools.ToRadians(camera_position[0]);
|
||||
}
|
||||
if (camera_position[1] !== null) {
|
||||
helperCamera.beta = BABYLON.Tools.ToRadians(camera_position[1]);
|
||||
}
|
||||
if (camera_position[2] !== null) {
|
||||
helperCamera.radius = camera_position[2];
|
||||
}
|
||||
helperCamera.lowerRadiusLimit = 0.1;
|
||||
const updateCameraSensibility = (): void => {
|
||||
helperCamera.wheelPrecision = 250 / (helperCamera.radius * zoom_speed);
|
||||
helperCamera.panningSensibility =
|
||||
(10000 * pan_speed) / helperCamera.radius;
|
||||
};
|
||||
updateCameraSensibility();
|
||||
helperCamera.attachControl(true);
|
||||
helperCamera.onAfterCheckInputsObservable.add(updateCameraSensibility);
|
||||
}
|
||||
|
||||
export function reset_camera_position(
|
||||
camera_position: [number | null, number | null, number | null],
|
||||
zoom_speed: number,
|
||||
pan_speed: number
|
||||
): void {
|
||||
if (scene) {
|
||||
scene.removeCamera(scene.activeCamera!);
|
||||
create_camera(scene, camera_position, zoom_speed, pan_speed);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<canvas bind:this={canvas}></canvas>
|
||||
@ -2,10 +2,7 @@
|
||||
import type { FileData } from "@gradio/client";
|
||||
import { BlockLabel, IconButton } from "@gradio/atoms";
|
||||
import { File, Download, Undo } from "@gradio/icons";
|
||||
import { add_new_model, reset_camera_position } from "./utils";
|
||||
import { onMount } from "svelte";
|
||||
import * as BABYLON from "babylonjs";
|
||||
import * as BABYLON_LOADERS from "babylonjs-loaders";
|
||||
import Canvas3D from "./Canvas3D.svelte";
|
||||
import type { I18nFormatter } from "@gradio/utils";
|
||||
import { dequal } from "dequal";
|
||||
|
||||
@ -25,71 +22,20 @@
|
||||
|
||||
let current_settings = { camera_position, zoom_speed, pan_speed };
|
||||
|
||||
$: {
|
||||
if (
|
||||
BABYLON_LOADERS.OBJFileLoader != undefined &&
|
||||
!BABYLON_LOADERS.OBJFileLoader.IMPORT_VERTEX_COLORS
|
||||
) {
|
||||
BABYLON_LOADERS.OBJFileLoader.IMPORT_VERTEX_COLORS = true;
|
||||
}
|
||||
}
|
||||
|
||||
let canvas: HTMLCanvasElement;
|
||||
let scene: BABYLON.Scene;
|
||||
let engine: BABYLON.Engine | null;
|
||||
let mounted = false;
|
||||
|
||||
onMount(() => {
|
||||
engine = new BABYLON.Engine(canvas, true);
|
||||
window.addEventListener("resize", () => {
|
||||
engine?.resize();
|
||||
});
|
||||
mounted = true;
|
||||
});
|
||||
|
||||
$: ({ path } = value || {
|
||||
path: undefined
|
||||
});
|
||||
|
||||
$: canvas && mounted && path && dispose();
|
||||
|
||||
function dispose(): void {
|
||||
if (scene && !scene.isDisposed) {
|
||||
scene.dispose();
|
||||
engine?.stopRenderLoop();
|
||||
engine?.dispose();
|
||||
engine = null;
|
||||
engine = new BABYLON.Engine(canvas, true);
|
||||
window.addEventListener("resize", () => {
|
||||
engine?.resize();
|
||||
});
|
||||
}
|
||||
if (engine !== null) {
|
||||
scene = add_new_model(
|
||||
canvas,
|
||||
scene,
|
||||
engine,
|
||||
value,
|
||||
clear_color,
|
||||
camera_position,
|
||||
zoom_speed,
|
||||
pan_speed
|
||||
);
|
||||
}
|
||||
}
|
||||
let canvas3d: Canvas3D;
|
||||
let resolved_url: string | undefined;
|
||||
|
||||
function handle_undo(): void {
|
||||
reset_camera_position(scene, camera_position, zoom_speed, pan_speed);
|
||||
canvas3d.reset_camera_position(camera_position, zoom_speed, pan_speed);
|
||||
}
|
||||
|
||||
$: {
|
||||
if (
|
||||
scene &&
|
||||
(!dequal(current_settings.camera_position, camera_position) ||
|
||||
current_settings.zoom_speed !== zoom_speed ||
|
||||
current_settings.pan_speed !== pan_speed)
|
||||
!dequal(current_settings.camera_position, camera_position) ||
|
||||
current_settings.zoom_speed !== zoom_speed ||
|
||||
current_settings.pan_speed !== pan_speed
|
||||
) {
|
||||
reset_camera_position(scene, camera_position, zoom_speed, pan_speed);
|
||||
canvas3d.reset_camera_position(camera_position, zoom_speed, pan_speed);
|
||||
current_settings = { camera_position, zoom_speed, pan_speed };
|
||||
}
|
||||
}
|
||||
@ -105,7 +51,7 @@
|
||||
<div class="buttons">
|
||||
<IconButton Icon={Undo} label="Undo" on:click={() => handle_undo()} />
|
||||
<a
|
||||
href={value.url}
|
||||
href={resolved_url}
|
||||
target={window.__is_colab__ ? "_blank" : null}
|
||||
download={window.__is_colab__ ? null : value.orig_name || value.path}
|
||||
>
|
||||
@ -113,7 +59,15 @@
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<canvas bind:this={canvas} />
|
||||
<Canvas3D
|
||||
bind:this={canvas3d}
|
||||
bind:resolved_url
|
||||
{value}
|
||||
{clear_color}
|
||||
{camera_position}
|
||||
{zoom_speed}
|
||||
{pan_speed}
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
@ -124,7 +78,7 @@
|
||||
width: var(--size-full);
|
||||
height: var(--size-full);
|
||||
}
|
||||
canvas {
|
||||
.model3D :global(canvas) {
|
||||
width: var(--size-full);
|
||||
height: var(--size-full);
|
||||
object-fit: contain;
|
||||
|
||||
@ -1,10 +1,11 @@
|
||||
<script lang="ts">
|
||||
import { createEventDispatcher, tick, onMount } from "svelte";
|
||||
import { createEventDispatcher, tick } from "svelte";
|
||||
import { Upload, ModifyUpload } from "@gradio/upload";
|
||||
import type { FileData } from "@gradio/client";
|
||||
import { BlockLabel } from "@gradio/atoms";
|
||||
import { File } from "@gradio/icons";
|
||||
import { add_new_model, reset_camera_position } from "./utils";
|
||||
import type { I18nFormatter } from "@gradio/utils";
|
||||
import Canvas3D from "./Canvas3D.svelte";
|
||||
|
||||
export let value: null | FileData;
|
||||
export let clear_color: [number, number, number, number] = [0, 0, 0, 0];
|
||||
@ -22,60 +23,25 @@
|
||||
null
|
||||
];
|
||||
|
||||
let mounted = false;
|
||||
let canvas: HTMLCanvasElement;
|
||||
let scene: BABYLON.Scene;
|
||||
let engine: BABYLON.Engine;
|
||||
|
||||
function reset_scene(): void {
|
||||
scene = add_new_model(
|
||||
canvas,
|
||||
scene,
|
||||
engine,
|
||||
value,
|
||||
clear_color,
|
||||
camera_position,
|
||||
zoom_speed,
|
||||
pan_speed
|
||||
);
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
if (value != null) {
|
||||
reset_scene();
|
||||
}
|
||||
mounted = true;
|
||||
});
|
||||
|
||||
$: ({ path } = value || {
|
||||
path: undefined
|
||||
});
|
||||
|
||||
$: canvas && mounted && path != null && reset_scene();
|
||||
|
||||
async function handle_upload({
|
||||
detail
|
||||
}: CustomEvent<FileData>): Promise<void> {
|
||||
value = detail;
|
||||
await tick();
|
||||
reset_scene();
|
||||
dispatch("change", value);
|
||||
dispatch("load", value);
|
||||
}
|
||||
|
||||
async function handle_clear(): Promise<void> {
|
||||
if (scene && engine) {
|
||||
scene.dispose();
|
||||
engine.dispose();
|
||||
}
|
||||
value = null;
|
||||
await tick();
|
||||
dispatch("clear");
|
||||
dispatch("change");
|
||||
}
|
||||
|
||||
let canvas3D: Canvas3D;
|
||||
async function handle_undo(): Promise<void> {
|
||||
reset_camera_position(scene, camera_position, zoom_speed, pan_speed);
|
||||
canvas3D.reset_camera_position(camera_position, zoom_speed, pan_speed);
|
||||
}
|
||||
|
||||
const dispatch = createEventDispatcher<{
|
||||
@ -87,19 +53,6 @@
|
||||
|
||||
let dragging = false;
|
||||
|
||||
import * as BABYLON from "babylonjs";
|
||||
import * as BABYLON_LOADERS from "babylonjs-loaders";
|
||||
import type { I18nFormatter } from "@gradio/utils";
|
||||
|
||||
$: {
|
||||
if (
|
||||
BABYLON_LOADERS.OBJFileLoader != undefined &&
|
||||
!BABYLON_LOADERS.OBJFileLoader.IMPORT_VERTEX_COLORS
|
||||
) {
|
||||
BABYLON_LOADERS.OBJFileLoader.IMPORT_VERTEX_COLORS = true;
|
||||
}
|
||||
}
|
||||
|
||||
$: dispatch("drag", dragging);
|
||||
</script>
|
||||
|
||||
@ -123,7 +76,14 @@
|
||||
on:undo={handle_undo}
|
||||
absolute
|
||||
/>
|
||||
<canvas bind:this={canvas} />
|
||||
<Canvas3D
|
||||
bind:this={canvas3D}
|
||||
{value}
|
||||
{clear_color}
|
||||
{camera_position}
|
||||
{zoom_speed}
|
||||
{pan_speed}
|
||||
></Canvas3D>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
@ -137,7 +97,7 @@
|
||||
height: var(--size-full);
|
||||
}
|
||||
|
||||
canvas {
|
||||
.input-model :global(canvas) {
|
||||
width: var(--size-full);
|
||||
height: var(--size-full);
|
||||
object-fit: contain;
|
||||
|
||||
@ -1,85 +0,0 @@
|
||||
import type { FileData } from "@gradio/client";
|
||||
import * as BABYLON from "babylonjs";
|
||||
|
||||
const create_camera = (
|
||||
scene: BABYLON.Scene,
|
||||
camera_position: [number | null, number | null, number | null],
|
||||
zoom_speed: number,
|
||||
pan_speed: number
|
||||
): void => {
|
||||
scene.createDefaultCamera(true, true, true);
|
||||
var helperCamera = scene.activeCamera! as BABYLON.ArcRotateCamera;
|
||||
if (camera_position[0] !== null) {
|
||||
helperCamera.alpha = BABYLON.Tools.ToRadians(camera_position[0]);
|
||||
}
|
||||
if (camera_position[1] !== null) {
|
||||
helperCamera.beta = BABYLON.Tools.ToRadians(camera_position[1]);
|
||||
}
|
||||
if (camera_position[2] !== null) {
|
||||
helperCamera.radius = camera_position[2];
|
||||
}
|
||||
helperCamera.lowerRadiusLimit = 0.1;
|
||||
const updateCameraSensibility = (): void => {
|
||||
helperCamera.wheelPrecision = 250 / (helperCamera.radius * zoom_speed);
|
||||
helperCamera.panningSensibility = (10000 * pan_speed) / helperCamera.radius;
|
||||
};
|
||||
updateCameraSensibility();
|
||||
helperCamera.attachControl(true);
|
||||
helperCamera.onAfterCheckInputsObservable.add(updateCameraSensibility);
|
||||
};
|
||||
|
||||
export const add_new_model = (
|
||||
canvas: HTMLCanvasElement,
|
||||
scene: BABYLON.Scene,
|
||||
engine: BABYLON.Engine,
|
||||
value: FileData | null,
|
||||
clear_color: [number, number, number, number],
|
||||
camera_position: [number | null, number | null, number | null],
|
||||
zoom_speed: number,
|
||||
pan_speed: number
|
||||
): BABYLON.Scene => {
|
||||
if (scene && !scene.isDisposed && engine) {
|
||||
scene.dispose();
|
||||
engine.dispose();
|
||||
}
|
||||
|
||||
engine = new BABYLON.Engine(canvas, true);
|
||||
scene = new BABYLON.Scene(engine);
|
||||
scene.createDefaultCameraOrLight();
|
||||
scene.clearColor = scene.clearColor = new BABYLON.Color4(...clear_color);
|
||||
|
||||
engine.runRenderLoop(() => {
|
||||
scene.render();
|
||||
});
|
||||
|
||||
window.addEventListener("resize", () => {
|
||||
engine.resize();
|
||||
});
|
||||
|
||||
if (!value) return scene;
|
||||
let url: string;
|
||||
|
||||
url = value.url!;
|
||||
|
||||
BABYLON.SceneLoader.ShowLoadingScreen = false;
|
||||
BABYLON.SceneLoader.Append(
|
||||
url,
|
||||
"",
|
||||
scene,
|
||||
() => create_camera(scene, camera_position, zoom_speed, pan_speed),
|
||||
undefined,
|
||||
undefined,
|
||||
"." + value.path.split(".")[1]
|
||||
);
|
||||
return scene;
|
||||
};
|
||||
|
||||
export const reset_camera_position = (
|
||||
scene: BABYLON.Scene,
|
||||
camera_position: [number | null, number | null, number | null],
|
||||
zoom_speed: number,
|
||||
pan_speed: number
|
||||
): void => {
|
||||
scene.removeCamera(scene.activeCamera!);
|
||||
create_camera(scene, camera_position, zoom_speed, pan_speed);
|
||||
};
|
||||
@ -1167,6 +1167,9 @@ importers:
|
||||
'@gradio/utils':
|
||||
specifier: workspace:^
|
||||
version: link:../utils
|
||||
'@gradio/wasm':
|
||||
specifier: workspace:^
|
||||
version: link:../wasm
|
||||
'@types/babylon':
|
||||
specifier: ^6.16.6
|
||||
version: 6.16.6
|
||||
|
||||
Loading…
Reference in New Issue
Block a user