Added chunked exporting for the structure blocks exporter

package-lock.json

@@ -41,6 +41,7 @@
         "jest": "^27.5.1",
         "jpeg-js": "^0.4.4",
         "json-loader": "^0.5.7",
+        "jszip": "^3.10.1",
         "node-polyfill-webpack-plugin": "^2.0.1",
         "pngjs": "^7.0.0",
         "prismarine-nbt": "^2.2.1",
@@ -5513,6 +5514,12 @@
       "dev": true,
       "hasInstallScript": true
     },
+    "node_modules/immediate": {
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
+      "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==",
+      "dev": true
+    },
     "node_modules/import-fresh": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
@@ -6680,6 +6687,18 @@
         "node": ">=6"
       }
     },
+    "node_modules/jszip": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",
+      "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==",
+      "dev": true,
+      "dependencies": {
+        "lie": "~3.3.0",
+        "pako": "~1.0.2",
+        "readable-stream": "~2.3.6",
+        "setimmediate": "^1.0.5"
+      }
+    },
     "node_modules/kind-of": {
       "version": "6.0.3",
       "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
@@ -6725,6 +6744,15 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/lie": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
+      "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
+      "dev": true,
+      "dependencies": {
+        "immediate": "~3.0.5"
+      }
+    },
     "node_modules/lines-and-columns": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
@@ -14971,6 +14999,12 @@
       "integrity": "sha512-Uq61/Q8XixCRyKvws7tPwboK0O70Dbr4kMQZHw2dqdEhnU6TpaGwyMg0vzQ4aaGtrO9N3etq46XwF7hxbqp8ug==",
       "dev": true
     },
+    "immediate": {
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
+      "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==",
+      "dev": true
+    },
     "import-fresh": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
@@ -15850,6 +15884,18 @@
       "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
       "dev": true
     },
+    "jszip": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",
+      "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==",
+      "dev": true,
+      "requires": {
+        "lie": "~3.3.0",
+        "pako": "~1.0.2",
+        "readable-stream": "~2.3.6",
+        "setimmediate": "^1.0.5"
+      }
+    },
     "kind-of": {
       "version": "6.0.3",
       "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
@@ -15883,6 +15929,15 @@
         "type-check": "~0.4.0"
       }
     },
+    "lie": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
+      "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
+      "dev": true,
+      "requires": {
+        "immediate": "~3.0.5"
+      }
+    },
     "lines-and-columns": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",

package.json

@@ -56,6 +56,7 @@
     "jest": "^27.5.1",
     "jpeg-js": "^0.4.4",
     "json-loader": "^0.5.7",
+    "jszip": "^3.10.1",
     "node-polyfill-webpack-plugin": "^2.0.1",
     "pngjs": "^7.0.0",
     "prismarine-nbt": "^2.2.1",

src/app_context.ts

@@ -12,7 +12,7 @@ import { AppConsole, TMessage } from './ui/console';
 import { UI } from './ui/layout';
 import { ColourSpace, EAction } from './util';
 import { ASSERT } from './util/error_util';
-import { download } from './util/file_util';
+import { download, downloadAsZip } from './util/file_util';
 import { LOG_ERROR, Logger } from './util/log_util';
 import { Vector3 } from './vector';
 import { WorkerController } from './worker_controller';
@@ -29,10 +29,12 @@ export class AppContext {
     private _lastAction?: EAction;
     public maxConstraint?: Vector3;
     private _materialManager: MaterialMapManager;
+    private _loadedFilename: string | null;
 
     private constructor() {
         this._workerController = new WorkerController();
         this._materialManager = new MaterialMapManager(new Map());
+        this._loadedFilename = null;
     }
 
     public static async init() {
@@ -83,10 +85,12 @@ export class AppContext {
         AppConsole.info(LOC('import.importing_mesh'));
         {
             // Instruct the worker to perform the job and await the result
+            const file = components.input.getValue();
+
             const resultImport = await this._workerController.execute({
                 action: 'Import',
                 params: {
-                    file: components.input.getValue(),
+                    file: file,
                     rotation: components.rotation.getValue(),
                 },
             });
@@ -104,6 +108,8 @@ export class AppContext {
                 .mulScalar(AppConfig.Get.CONSTRAINT_MAXIMUM_HEIGHT / 8.0).floor();
             this._materialManager = new MaterialMapManager(resultImport.result.materials);
             UI.Get.updateMaterialsAction(this._materialManager);
+
+            this._loadedFilename = file.name.split('.')[0] ?? 'result';
         }
 
         AppConsole.info(LOC('import.rendering_mesh'));
@@ -305,7 +311,19 @@ export class AppContext {
             ASSERT(resultExport.action === 'Export');
 
             this._addWorkerMessagesToConsole(resultExport.messages);
-            download(resultExport.result.buffer, 'result.' + resultExport.result.extension);
+
+            ASSERT(this._loadedFilename !== null)
+            const fileExport = resultExport.result.files;
+            if (fileExport.type === 'single') {
+                download(fileExport.content, `${this._loadedFilename}_OTS${fileExport.extension}`);
+            } else {
+                const zipFiles = fileExport.regions.map((region) => {
+                    // .nbt exports need to be lowercase
+                    return { content: region.content, filename: `ots_${region.name}${fileExport.extension}` }
+                });
+
+                downloadAsZip(`${this._loadedFilename}_OTS.zip`, zipFiles);
+            }
         }
         AppConsole.success(LOC('export.exported_structure'));
 

src/bounds.ts

@@ -22,6 +22,7 @@ export class Bounds {
         this._max = Vector3.max(this._max, volume._max);
     }
 
+    // TODO: rename to `createInfinitesimalBounds`
     public static getInfiniteBounds() {
         return new Bounds(
             new Vector3(Infinity, Infinity, Infinity),
@@ -37,11 +38,13 @@ export class Bounds {
         return this._max;
     }
 
+    // TODO: Rename to `calcCentre`
     public getCentre() {
         const extents = Vector3.sub(this._max, this._min).divScalar(2);
         return Vector3.add(this.min, extents);
     }
 
+    // TODO: Rename to `calcDimensions`
     public getDimensions() {
         return Vector3.sub(this._max, this._min);
     }

src/config.ts

@@ -11,7 +11,7 @@ export class AppConfig {
     public readonly RELEASE_MODE = true;
     public readonly MAJOR_VERSION = 0;
     public readonly MINOR_VERSION = 8;
-    public readonly HOTFIX_VERSION = 4;
+    public readonly HOTFIX_VERSION = 5;
     public readonly VERSION_TYPE: 'd' | 'a' | 'r' = 'r'; // dev, alpha, or release build
     public readonly MINECRAFT_VERSION = '1.19.4';
 

src/exporters/base_exporter.ts

@@ -1,5 +1,11 @@
 import { BlockMesh } from '../block_mesh';
 
+export type TStructureRegion = { name: string, content: Buffer };
+
+export type TStructureExport =
+    | { type: 'single', extension: string, content: Buffer }
+    | { type: 'multiple', extension: string, regions: TStructureRegion[] }
+
 export abstract class IExporter {
     /** The file type extension of this exporter.
      * @note Do not include the dot prefix, e.g. 'obj' not '.obj'.
@@ -14,5 +20,5 @@ export abstract class IExporter {
      * @param blockMesh The block mesh to export.
      * @param filePath The location to save the file to.
      */
-    public abstract export(blockMesh: BlockMesh): Buffer;
+    public abstract export(blockMesh: BlockMesh): TStructureExport;
 }

src/exporters/indexed_json_exporter .ts

@@ -1,5 +1,5 @@
 import { BlockMesh } from '../block_mesh';
-import { IExporter } from './base_exporter';
+import { IExporter, TStructureExport } from './base_exporter';
 
 export class IndexedJSONExporter extends IExporter {
     public override getFormatFilter() {
@@ -9,7 +9,7 @@ export class IndexedJSONExporter extends IExporter {
         };
     }
 
-    public override export(blockMesh: BlockMesh): Buffer {
+    public override export(blockMesh: BlockMesh): TStructureExport {
         const blocks = blockMesh.getBlocks();
 
         const blocksUsed = blockMesh.getBlockPalette();
@@ -34,6 +34,6 @@ export class IndexedJSONExporter extends IExporter {
             xyzi: blockArray,
         });
 
-        return Buffer.from(json);
+        return { type: 'single', extension: '.json', content: Buffer.from(json) };
     }
 }

src/exporters/litematic_exporter.ts

@@ -7,7 +7,8 @@ import { AppTypes } from '../util';
 import { ASSERT } from '../util/error_util';
 import { saveNBT } from '../util/nbt_util';
 import { Vector3 } from '../vector';
-import { IExporter } from './base_exporter';
+import { IExporter, TStructureExport } from './base_exporter';
+import { save } from '@loaders.gl/core';
 
 type BlockID = number;
 type long = [number, number];
@@ -21,9 +22,9 @@ export class Litematic extends IExporter {
         };
     }
 
-    public override export(blockMesh: BlockMesh) {
+    public override export(blockMesh: BlockMesh): TStructureExport {
         const nbt = this._convertToNBT(blockMesh);
-        return saveNBT(nbt);
+        return { type: 'single', extension: '.litematic', content: saveNBT(nbt) };
     }
 
     /**

src/exporters/nbt_exporter.ts

@@ -7,7 +7,9 @@ import { StatusHandler } from '../status';
 import { AppUtil } from '../util';
 import { saveNBT } from '../util/nbt_util';
 import { Vector3 } from '../vector';
-import { IExporter } from './base_exporter';
+import { IExporter, TStructureExport, TStructureRegion } from './base_exporter';
+import { Bounds } from '../bounds';
+import { ASSERT } from '../util/error_util';
 
 export class NBTExporter extends IExporter {
     public override getFormatFilter() {
@@ -17,39 +19,25 @@ export class NBTExporter extends IExporter {
         };
     }
 
-    public override export(blockMesh: BlockMesh) {
+    private _processChunk(blockMesh: BlockMesh, min: Vector3, blockNameToIndex: Map<string, number>, palette: any): Buffer {
-        const bounds = blockMesh.getVoxelMesh().getBounds();
+        const blocks: any[] = [];
-        const sizeVector = bounds.getDimensions().add(1);
-
-        const isTooBig = sizeVector.x > 48 && sizeVector.y > 48 && sizeVector.z > 48;
-        if (isTooBig) {
-            StatusHandler.warning(LOC('export.nbt_exporter_too_big'));
-        }
-
-        const blockNameToIndex = new Map<string, number>();
-        const palette: any = [];
-        for (const blockName of blockMesh.getBlockPalette()) {
-            palette.push({
-                Name: {
-                    type: TagType.String,
-                    value: AppUtil.Text.namespaceBlock(blockName),
-                },
-            });
-            blockNameToIndex.set(blockName, palette.length - 1);
-        }
-
-        const blocks: any = [];
         for (const block of blockMesh.getBlocks()) {
             const pos = block.voxel.position;
             const blockIndex = blockNameToIndex.get(block.blockInfo.name);
+
             if (blockIndex !== undefined) {
-                if (pos.x > -24 && pos.x <= 24 && pos.y > -24 && pos.y <= 24 && pos.z > -24 && pos.z <= 24) {
+                if (pos.x >= min.x && pos.x < min.x + 48 && pos.y >= min.y && pos.y < min.y + 48 && pos.z >= min.z && pos.z < min.z + 48) {
+                    const translatedPos = Vector3.sub(block.voxel.position, min);
+                    ASSERT(translatedPos.x >= 0 && translatedPos.x < 48);
+                    ASSERT(translatedPos.y >= 0 && translatedPos.y < 48);
+                    ASSERT(translatedPos.z >= 0 && translatedPos.z < 48);
+
                     blocks.push({
                         pos: {
                             type: TagType.List,
                             value: {
                                 type: TagType.Int,
-                                value: Vector3.sub(block.voxel.position, bounds.min).toArray(),
+                                value: translatedPos.toArray(),
                             },
                         },
                         state: {
@@ -60,6 +48,7 @@ export class NBTExporter extends IExporter {
                 }
             }
         }
+        ASSERT(blocks.length < 48 * 48 * 48);
 
         const nbt: NBT = {
             type: TagType.Compound,
@@ -73,7 +62,7 @@ export class NBTExporter extends IExporter {
                     type: TagType.List,
                     value: {
                         type: TagType.Int,
-                        value: sizeVector.toArray(),
+                        value: [48, 48, 48],
                     },
                 },
                 palette: {
@@ -95,4 +84,47 @@ export class NBTExporter extends IExporter {
 
         return saveNBT(nbt);
     }
+
+    public override export(blockMesh: BlockMesh) {
+        const bounds = blockMesh.getVoxelMesh().getBounds();
+        /*
+        const sizeVector = bounds.getDimensions().add(1);
+
+        const isTooBig = sizeVector.x > 48 && sizeVector.y > 48 && sizeVector.z > 48;
+        if (isTooBig) {
+            StatusHandler.warning(LOC('export.nbt_exporter_too_big'));
+        }
+        */
+
+        const blockNameToIndex = new Map<string, number>();
+        const palette: any = [];
+        for (const blockName of blockMesh.getBlockPalette()) {
+            palette.push({
+                Name: {
+                    type: TagType.String,
+                    value: AppUtil.Text.namespaceBlock(blockName),
+                },
+            });
+            blockNameToIndex.set(blockName, palette.length - 1);
+        }
+
+        const regions: TStructureRegion[] = [];
+
+        for (let x = bounds.min.x; x < bounds.max.x; x += 48) {
+            for (let y = bounds.min.y; y < bounds.max.y; y += 48) {
+                for (let z = bounds.min.z; z < bounds.max.z; z += 48) {
+                    const buffer = this._processChunk(blockMesh, new Vector3(x, y, z), blockNameToIndex, palette);
+                    regions.push({ content: buffer, name: `x${x - bounds.min.x}_y${y - bounds.min.y}_z${z - bounds.min.z}` });
+                }
+            }
+        }
+
+        const out: TStructureExport = {
+            type: 'multiple',
+            extension: '.nbt',
+            regions: regions
+        }
+
+        return out;
+    }
 }

src/exporters/schem_exporter.ts

@@ -7,7 +7,7 @@ import { LOG } from '../util/log_util';
 import { MathUtil } from '../util/math_util';
 import { saveNBT } from '../util/nbt_util';
 import { Vector3 } from '../vector';
-import { IExporter } from './base_exporter';
+import { IExporter, TStructureExport } from './base_exporter';
 
 export class SchemExporter extends IExporter {
     private static SCHEMA_VERSION = 2;
@@ -19,7 +19,7 @@ export class SchemExporter extends IExporter {
         };
     }
 
-    public override export(blockMesh: BlockMesh) {
+    public override export(blockMesh: BlockMesh): TStructureExport {
         const bounds = blockMesh.getVoxelMesh().getBounds();
         const sizeVector = bounds.getDimensions().add(1);
 
@@ -77,7 +77,7 @@ export class SchemExporter extends IExporter {
             },
         };
 
-        return saveNBT(nbt);
+        return { type: 'single', extension: '.schem', content: saveNBT(nbt) };
     }
 
     private static _getBufferIndex(dimensions: Vector3, vec: Vector3) {

src/exporters/schematic_exporter.ts

@@ -9,7 +9,7 @@ import { StatusHandler } from '../status';
 import { LOG_WARN } from '../util/log_util';
 import { saveNBT } from '../util/nbt_util';
 import { Vector3 } from '../vector';
-import { IExporter } from './base_exporter';
+import { IExporter, TStructureExport } from './base_exporter';
 
 export class Schematic extends IExporter {
     public override getFormatFilter() {
@@ -19,9 +19,9 @@ export class Schematic extends IExporter {
         };
     }
 
-    public override export(blockMesh: BlockMesh) {
+    public override export(blockMesh: BlockMesh): TStructureExport {
         const nbt = this._convertToNBT(blockMesh);
-        return saveNBT(nbt);
+        return { type: 'single', extension: '.schematic', content: saveNBT(nbt) };
     }
 
     private _convertToNBT(blockMesh: BlockMesh): NBT {

src/exporters/uncompressed_json_exporter.ts

@@ -1,5 +1,5 @@
 import { BlockMesh } from '../block_mesh';
-import { IExporter } from './base_exporter';
+import { IExporter, TStructureExport } from './base_exporter';
 
 export class UncompressedJSONExporter extends IExporter {
     public override getFormatFilter() {
@@ -9,7 +9,7 @@ export class UncompressedJSONExporter extends IExporter {
         };
     }
 
-    public override export(blockMesh: BlockMesh): Buffer {
+    public override export(blockMesh: BlockMesh): TStructureExport {
         const blocks = blockMesh.getBlocks();
 
         const lines = new Array<string>();
@@ -33,6 +33,6 @@ export class UncompressedJSONExporter extends IExporter {
 
         const json = lines.join('');
 
-        return Buffer.from(json);
+        return { type: 'single', extension: '.json', content: Buffer.from(json) };
     }
 }

src/util/file_util.ts

@@ -1,9 +1,24 @@
+import { TStructureExport } from "../exporters/base_exporter";
+import JSZip, { file } from 'jszip';
+
 export function download(content: any, filename: string) {
     const a = document.createElement('a'); // Create "a" element
     const blob = new Blob([content]); // Create a blob (file-like object)
     const url = URL.createObjectURL(blob); // Create an object URL from blob
-    
+
     a.setAttribute('href', url); // Set "a" element link
     a.setAttribute('download', filename); // Set download filename
     a.click();
 }
+
+export function downloadAsZip(zipFilename: string, files: { content: any, filename: string }[]) {
+    const zip = new JSZip();
+
+    files.forEach((file) => {
+        zip.file(file.filename, file.content);
+    });
+
+    zip.generateAsync({type:"blob"}).then(function(content) {
+        download(content, zipFilename);
+    });
+}

src/worker_client.ts

@@ -228,11 +228,10 @@ export class WorkerClient {
         ASSERT(this._loadedBlockMesh !== undefined);
 
         const exporter: IExporter = ExporterFactory.GetExporter(params.exporter);
-        const buffer = exporter.export(this._loadedBlockMesh);
+        const files = exporter.export(this._loadedBlockMesh);
 
         return {
-            buffer: buffer,
+            files: files,
-            extension: exporter.getFormatFilter().extension,
         };
     }
 }

src/worker_types.ts

@@ -2,6 +2,7 @@ import { FallableBehaviour } from './block_mesh';
 import { Bounds } from './bounds';
 import { TBlockMeshBufferDescription, TMeshBufferDescription, TVoxelMeshBufferDescription } from './buffer';
 import { RGBAUtil } from './colour';
+import { TStructureExport } from './exporters/base_exporter';
 import { TExporters } from './exporters/exporters';
 import { MaterialMap } from './mesh';
 import { TMessage } from './ui/console';
@@ -169,8 +170,7 @@ export namespace ExportParams {
     }
 
     export type Output = {
-        buffer: Buffer,
+        files: TStructureExport
-        extension: string,
     }
 }