diff --git a/README.md b/README.md index 59290a8a..690e85af 100644 --- a/README.md +++ b/README.md @@ -22,13 +22,22 @@ When I started, web-based viewers were already available -- A WebGL-based viewer - WASM splat sort: Implemented in C++ using WASM SIMD instructions - Partially GPU accelerated splat sort: Uses transform feedback to pre-calculate splat distances +## Tips + +- Progressively loaded `.ply` and `.splat` files will not have certain optimizations such as cache-optimized splat ordering applied to them. For optimial performance, convert these file types to `.ksplat` or load them non-progressively. +- Converting your scenes to `.ksplat` will result in the fastest loading times since its format matches the internal format for splat data. +- Scenes with large dimensions or high splat density will cause issues with the default settings. For those scenes, you can try a couple of things: + - Set the viewer parameter `integerBasedSort` to `false` to force a slower, floating-point based splat sort. + - Experiment with a larger value for viewer parameter `splatSortDistanceMapPrecision`, to adjust the precision for the distance map in the splat sort. Larger precision values will result in reduced performance, but often can alleviate visual artifacts that arise when the precision is too low. + + ## Known issues - Splat sort runs on the CPU – would be great to figure out a GPU-based approach - Artifacts are visible when you move or rotate too fast (due to CPU-based splat sort) - Sub-optimal performance on mobile devices - Custom `.ksplat` file format still needs work, especially around compression -- The default, integer based splat sort does not work well for larger scenes. In that case a value of `false` for the `integerBasedSort` viewer parameter can force a slower, floating-point based sort +- Scenes with very large dimensions will probably crash (often with an `Index out of bounds` error from the splat sort). Changing `splatSortDistanceMapPrecision` or `integerBasedSort` will probably not help in those cases. ## Limitations @@ -288,7 +297,7 @@ const viewer = new GaussianSplats3D.Viewer({ 'logLevel': GaussianSplats3D.LogLevel.None, 'sphericalHarmonicsDegree': 0, `enableOptionalEffects`: false, - `plyInMemoryCompressionLevel`: 2 + `inMemoryCompressionLevel`: 2 `freeIntermediateSplatData`: false }); viewer.addSplatScene('') @@ -317,9 +326,11 @@ Advanced `Viewer` parameters | `enableSIMDInSort` | Enable the usage of SIMD WebAssembly instructions for the splat sort. Default is `true`. | `sharedMemoryForWorkers` | Tells the viewer to use shared memory via a `SharedArrayBuffer` to transfer data to and from the sorting web worker. If set to `false`, it is recommended that `gpuAcceleratedSort` be set to `false` as well. Defaults to `true`. | `integerBasedSort` | Tells the sorting web worker to use the integer versions of relevant data to compute the distance of splats from the camera. Since integer arithmetic is faster than floating point, this reduces sort time. However it can result in integer overflows in larger scenes so it should only be used for small scenes. Defaults to `true`. +| `splatSortDistanceMapPrecision` | Specify the precision for the distance map used in the splat sort algorithm. Defaults to 16 (16-bit). A lower precision is faster, but may result in visual artifacts in larger or denser scenes. | `halfPrecisionCovariancesOnGPU` | Tells the viewer to use 16-bit floating point values when storing splat covariance data in textures, instead of 32-bit. Defaults to `false`. | `dynamicScene` | Tells the viewer to not make any optimizations that depend on the scene being static. Additionally all splat data retrieved from the viewer's splat mesh will not have their respective scene transform applied to them by default. | `webXRMode` | Tells the viewer whether or not to enable built-in Web VR or Web AR. Valid values are defined in the `WebXRMode` enum: `None`, `VR`, and `AR`. Defaults to `None`. +| `webXRSessionInit` | Tells the viewer to build a WebXR session with some options. Defaults with {}. For more details : https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/requestSession#options | `renderMode` | Controls when the viewer renders the scene. Valid values are defined in the `RenderMode` enum: `Always`, `OnChange`, and `Never`. Defaults to `Always`. | `sceneRevealMode` | Controls the fade-in effect used when the scene is loaded. Valid values are defined in the `SceneRevealMode` enum: `Default`, `Gradual`, and `Instant`. `Default` results in a nice, slow fade-in effect for progressively loaded scenes, and a fast fade-in for non progressively loaded scenes. `Gradual` will force a slow fade-in for all scenes. `Instant` will force all loaded scene data to be immediately visible. | `antialiased` | When true, will perform additional steps during rendering to address artifacts caused by the rendering of gaussians at substantially different resolutions than that at which they were rendered during training. This will only work correctly for models that were trained using a process that utilizes this compensation calculation. For more details: https://github.com/nerfstudio-project/gsplat/pull/117, https://github.com/graphdeco-inria/gaussian-splatting/issues/294#issuecomment-1772688093 @@ -327,9 +338,11 @@ Advanced `Viewer` parameters | `logLevel` | Verbosity of the console logging. Defaults to `GaussianSplats3D.LogLevel.None`. | `sphericalHarmonicsDegree` | Degree of spherical harmonics to utilize in rendering splats (assuming the data is present in the splat scene). Valid values are 0, 1, or 2. Default value is 0. | `enableOptionalEffects` | When true, allows for usage of extra properties and attributes during rendering for effects such as opacity adjustment. Default is `false` for performance reasons. These properties are separate from transform properties (scale, rotation, position) that are enabled by the `dynamicScene` parameter. -| `plyInMemoryCompressionLevel` | Level to compress `.ply` files when loading them for direct rendering (not exporting to `.ksplat`). Valid values are the same as `.ksplat` compression levels (0, 1, or 2). Default is 2. +| `inMemoryCompressionLevel` | Level to compress `.ply` or `.ksplat` files when loading them for direct rendering (not exporting to `.ksplat`). Valid values are the same as `.ksplat` compression levels (0, 1, or 2). Default is 0. +| `optimizeSplatData` | Reorder splat data in memory after loading is complete to optimize cache utilization. Default is `true`. Does not apply if splat scene is progressively loaded. | `freeIntermediateSplatData` | When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and used to populate data textures will be freed. This will reduces memory usage, but if that data needs to be modified it will need to be re-populated from the splat buffer(s). Defaults to `false`. | `splatRenderMode` | Determine which splat rendering mode to enable. Valid values are defined in the `SplatRenderMode` enum: `ThreeD` and `TwoD`. `ThreeD` is the original/traditional mode and `TwoD` is the new mode described here: https://surfsplatting.github.io/ +| `sceneFadeInRateMultiplier` | Customize the speed at which the scene is revealed. Default is 1.0.
### Creating KSPLAT files @@ -342,8 +355,12 @@ const compressionLevel = 1; const splatAlphaRemovalThreshold = 5; // out of 255 const sphericalHarmonicsDegree = 1; GaussianSplats3D.PlyLoader.loadFromURL('', + onProgress, + progressiveLoad, + onProgressiveLoadSectionProgress, + minimumAlpha, compressionLevel, - splatAlphaRemovalThreshold, + optimizeSplatData, sphericalHarmonicsDegree) .then((splatBuffer) => { GaussianSplats3D.KSplatLoader.downloadFile(splatBuffer, 'converted_file.ksplat'); diff --git a/demo/index.html b/demo/index.html index 29916fb9..2810f68d 100644 --- a/demo/index.html +++ b/demo/index.html @@ -294,10 +294,10 @@ function fileBufferToSplatBuffer(fileBufferData, format, alphaRemovalThreshold, compressionLevel, sectionSize, sceneCenter, blockSize, bucketSize, outSphericalHarmonicsDegree = 0) { if (format === GaussianSplats3D.SceneFormat.Ply) { - return GaussianSplats3D.PlyLoader.loadFromFileData(fileBufferData.data, alphaRemovalThreshold, compressionLevel, outSphericalHarmonicsDegree, sectionSize, sceneCenter, blockSize, bucketSize); + return GaussianSplats3D.PlyLoader.loadFromFileData(fileBufferData.data, alphaRemovalThreshold, compressionLevel, true, outSphericalHarmonicsDegree, sectionSize, sceneCenter, blockSize, bucketSize); } else { if (format === GaussianSplats3D.SceneFormat.Splat) { - return GaussianSplats3D.SplatLoader.loadFromFileData(fileBufferData.data, alphaRemovalThreshold, compressionLevel, sectionSize, sceneCenter, blockSize, bucketSize); + return GaussianSplats3D.SplatLoader.loadFromFileData(fileBufferData.data, alphaRemovalThreshold, compressionLevel, true, sectionSize, sceneCenter, blockSize, bucketSize); } else { return GaussianSplats3D.KSplatLoader.loadFromFileData(fileBufferData.data); } diff --git a/package-lock.json b/package-lock.json index 7b829a9e..decfc9e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.4", + "version": "0.4.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.4", + "version": "0.4.5", "license": "MIT", "devDependencies": { "@babel/core": "7.22.0", diff --git a/package.json b/package.json index d899299e..ff08960f 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "type": "git", "url": "https://github.com/mkkellogg/GaussianSplats3D" }, - "version": "0.4.4", + "version": "0.4.5", "description": "Three.js-based 3D Gaussian splat viewer", "module": "build/gaussian-splats-3d.module.js", "main": "build/gaussian-splats-3d.umd.cjs", diff --git a/src/ArrowHelper.js b/src/ArrowHelper.js index ce309115..1ac163b4 100644 --- a/src/ArrowHelper.js +++ b/src/ArrowHelper.js @@ -1,5 +1,7 @@ import * as THREE from 'three'; +const _axis = new THREE.Vector3(); + export class ArrowHelper extends THREE.Object3D { constructor(dir = new THREE.Vector3(0, 0, 1), origin = new THREE.Vector3(0, 0, 0), length = 1, diff --git a/src/Constants.js b/src/Constants.js index cec02275..1694d2eb 100644 --- a/src/Constants.js +++ b/src/Constants.js @@ -1,6 +1,6 @@ export class Constants { - static DepthMapRange = 1 << 16; + static DefaultSplatSortDistanceMapPrecision = 16; static MemoryPageSize = 65536; static BytesPerFloat = 4; static BytesPerInt = 4; diff --git a/src/DropInViewer.js b/src/DropInViewer.js index b56c2c94..5db045f5 100644 --- a/src/DropInViewer.js +++ b/src/DropInViewer.js @@ -112,8 +112,12 @@ export class DropInViewer extends THREE.Group { return this.viewer.getSceneCount(); } - dispose() { - return this.viewer.dispose(); + setActiveSphericalHarmonicsDegrees(activeSphericalHarmonicsDegrees) { + this.viewer.setActiveSphericalHarmonicsDegrees(activeSphericalHarmonicsDegrees); + } + + async dispose() { + return await this.viewer.dispose(); } static onBeforeRender(viewer, renderer, threeScene, camera) { diff --git a/src/Util.js b/src/Util.js index 4e0a423b..870dbe6c 100644 --- a/src/Util.js +++ b/src/Util.js @@ -60,13 +60,20 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { const signal = abortController.signal; let aborted = false; const abortHandler = (reason) => { - abortController.abort(new AbortedPromiseError(reason)); + abortController.abort(reason); aborted = true; }; return new AbortablePromise((resolve, reject) => { fetch(path, { signal }) .then(async (data) => { + // Handle error conditions where data is still returned + if (!data.ok) { + const errorText = await data.text(); + reject(new Error(`Fetch failed: ${data.status} ${data.statusText} ${errorText}`)); + return; + } + const reader = data.body.getReader(); let bytesDownloaded = 0; let _fileSize = data.headers.get('Content-Length'); @@ -96,19 +103,20 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { percent = bytesDownloaded / fileSize * 100; percentLabel = `${percent.toFixed(2)}%`; } - if (saveChunks) chunks.push(chunk); + if (saveChunks) { + chunks.push(chunk); + } if (onProgress) { - const cancelSaveChucnks = onProgress(percent, percentLabel, chunk, fileSize); - if (cancelSaveChucnks) saveChunks = false; + onProgress(percent, percentLabel, chunk, fileSize); } } catch (error) { reject(error); - break; + return; } } }) .catch((error) => { - reject(error); + reject(new AbortedPromiseError(error)); }); }, abortHandler); diff --git a/src/Viewer.js b/src/Viewer.js index 244cd914..548fc01b 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -12,7 +12,7 @@ import { Raycaster } from './raycaster/Raycaster.js'; import { SplatMesh } from './splatmesh/SplatMesh.js'; import { createSortWorker } from './worker/SortWorker.js'; import { Constants } from './Constants.js'; -import { getCurrentTime, isIOS, getIOSSemever } from './Util.js'; +import { getCurrentTime, isIOS, getIOSSemever, clamp } from './Util.js'; import { AbortablePromise, AbortedPromiseError } from './AbortablePromise.js'; import { SceneFormat } from './loaders/SceneFormat.js'; import { WebXRMode } from './webxr/WebXRMode.js'; @@ -20,6 +20,7 @@ import { VRButton } from './webxr/VRButton.js'; import { ARButton } from './webxr/ARButton.js'; import { delayedExecute, abortablePromiseWithExtractedComponents } from './Util.js'; import { LoaderStatus } from './loaders/LoaderStatus.js'; +import { DirectLoadError } from './loaders/DirectLoadError.js'; import { RenderMode } from './RenderMode.js'; import { LogLevel } from './LogLevel.js'; import { SceneRevealMode } from './SceneRevealMode.js'; @@ -122,6 +123,8 @@ export class Viewer { } this.webXRActive = false; + this.webXRSessionInit = options.webXRSessionInit || {}; + // if 'renderMode' is RenderMode.Always, then the viewer will rrender the scene on every update. If it is RenderMode.OnChange, // it will only render when something in the scene has changed. this.renderMode = options.renderMode || RenderMode.Always; @@ -155,11 +158,18 @@ export class Viewer { if (options.enableSIMDInSort === undefined || options.enableSIMDInSort === null) options.enableSIMDInSort = true; this.enableSIMDInSort = options.enableSIMDInSort; - // Level to compress PLY files when loading them for direct rendering (not exporting to .ksplat) - if (options.plyInMemoryCompressionLevel === undefined || options.plyInMemoryCompressionLevel === null) { - options.plyInMemoryCompressionLevel = 2; + // Level to compress non KSPLAT files when loading them for direct rendering + if (options.inMemoryCompressionLevel === undefined || options.inMemoryCompressionLevel === null) { + options.inMemoryCompressionLevel = 0; + } + this.inMemoryCompressionLevel = options.inMemoryCompressionLevel; + + // Reorder splat data in memory after loading is complete to optimize cache utilization. Default is true. + // Does not apply if splat scene is progressively loaded. + if (options.optimizeSplatData === undefined || options.optimizeSplatData === null) { + options.optimizeSplatData = true; } - this.plyInMemoryCompressionLevel = options.plyInMemoryCompressionLevel; + this.optimizeSplatData = options.optimizeSplatData; // When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and is used to // populate the data textures will be freed. This will reduces memory usage, but if that data needs to be modified @@ -187,6 +197,14 @@ export class Viewer { } this.splatRenderMode = options.splatRenderMode; + // Customize the speed at which the scene is revealed + this.sceneFadeInRateMultiplier = options.sceneFadeInRateMultiplier || 1.0; + + // Set the range for the depth map for the counting sort used to sort the splats + this.splatSortDistanceMapPrecision = options.splatSortDistanceMapPrecision || Constants.DefaultSplatSortDistanceMapPrecision; + const maxPrecision = this.integerBasedSort ? 20 : 24; + this.splatSortDistanceMapPrecision = clamp(this.splatSortDistanceMapPrecision, 10, maxPrecision); + this.onSplatMeshChangedCallback = null; this.createSplatMesh(); @@ -206,10 +224,13 @@ export class Viewer { this.sortWorker = null; this.sortRunning = false; this.splatRenderCount = 0; + this.splatSortCount = 0; + this.lastSplatSortCount = 0; this.sortWorkerIndexesToSort = null; this.sortWorkerSortedIndexes = null; this.sortWorkerPrecomputedDistances = null; this.sortWorkerTransforms = null; + this.preSortMessages = []; this.runAfterNextSort = []; this.selfDrivenModeRunning = false; @@ -265,7 +286,7 @@ export class Viewer { this.splatMesh = new SplatMesh(this.splatRenderMode, this.dynamicScene, this.enableOptionalEffects, this.halfPrecisionCovariancesOnGPU, this.devicePixelRatio, this.gpuAcceleratedSort, this.integerBasedSort, this.antialiased, this.maxScreenSpaceSplatSize, this.logLevel, - this.sphericalHarmonicsDegree); + this.sphericalHarmonicsDegree, this.sceneFadeInRateMultiplier); this.splatMesh.frustumCulled = false; if (this.onSplatMeshChangedCallback) this.onSplatMeshChangedCallback(); } @@ -288,7 +309,7 @@ export class Viewer { this.setupCamera(); this.setupRenderer(); - this.setupWebXR(); + this.setupWebXR(this.webXRSessionInit); this.setupControls(); this.setupEventHandlers(); @@ -345,12 +366,12 @@ export class Viewer { } - setupWebXR() { + setupWebXR(webXRSessionInit) { if (this.webXRMode) { if (this.webXRMode === WebXRMode.VR) { - this.rootElement.appendChild(VRButton.createButton(this.renderer)); + this.rootElement.appendChild(VRButton.createButton(this.renderer, webXRSessionInit)); } else if (this.webXRMode === WebXRMode.AR) { - this.rootElement.appendChild(ARButton.createButton(this.renderer)); + this.rootElement.appendChild(ARButton.createButton(this.renderer, webXRSessionInit)); } this.renderer.xr.addEventListener('sessionstart', (e) => { this.webXRActive = true; @@ -424,6 +445,11 @@ export class Viewer { this.renderMode = renderMode; } + setActiveSphericalHarmonicsDegrees(activeSphericalHarmonicsDegrees) { + this.splatMesh.material.uniforms.sphericalHarmonicsDegree.value = activeSphericalHarmonicsDegrees; + this.splatMesh.material.uniformsNeedUpdate = true; + } + onSplatMeshChanged(callback) { this.onSplatMeshChangedCallback = callback; } @@ -863,7 +889,6 @@ export class Viewer { .then(() => { progressiveLoadedSectionBuilding = false; if (queuedBuild.firstBuild) { - progressiveLoadFirstSectionBuildPromise.reject = null; progressiveLoadFirstSectionBuildPromise.resolve(); } else if (queuedBuild.finalBuild) { splatSceneDownloadAndBuildPromise.resolve(); @@ -891,8 +916,8 @@ export class Viewer { } }; - let splatSceneDownloadPromise = this.downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold, onDownloadProgress, true, - onProgressiveLoadSectionProgress, format); + const splatSceneDownloadPromise = this.downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold, onDownloadProgress, true, + onProgressiveLoadSectionProgress, format); const progressiveLoadFirstSectionBuildPromise = abortablePromiseWithExtractedComponents(splatSceneDownloadPromise.abortHandler); const splatSceneDownloadAndBuildPromise = abortablePromiseWithExtractedComponents(); @@ -1023,14 +1048,26 @@ export class Viewer { */ downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold = 1, onProgress = undefined, progressiveBuild = false, onSectionBuilt = undefined, format) { - if (format === SceneFormat.Splat) { - return SplatLoader.loadFromURL(path, onProgress, progressiveBuild, - onSectionBuilt, splatAlphaRemovalThreshold, 0, false); - } else if (format === SceneFormat.KSplat) { - return KSplatLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt); - } else if (format === SceneFormat.Ply) { - return PlyLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, - splatAlphaRemovalThreshold, this.plyInMemoryCompressionLevel, this.sphericalHarmonicsDegree); + + const optimizeSplatData = progressiveBuild ? false : this.optimizeSplatData; + try { + if (format === SceneFormat.Splat) { + return SplatLoader.loadFromURL(path, onProgress, progressiveBuild, + onSectionBuilt, splatAlphaRemovalThreshold, + this.inMemoryCompressionLevel, optimizeSplatData); + } else if (format === SceneFormat.KSplat) { + return KSplatLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt); + } else if (format === SceneFormat.Ply) { + return PlyLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, + splatAlphaRemovalThreshold, this.inMemoryCompressionLevel, + optimizeSplatData, this.sphericalHarmonicsDegree); + } + } catch (e) { + if (e instanceof DirectLoadError) { + throw new Error('File type or server does not support progressive loading.'); + } else { + throw e; + } } throw new Error(`Viewer::downloadSplatSceneToSplatBuffer -> File format not supported: ${path}`); @@ -1052,9 +1089,7 @@ export class Viewer { if (this.isDisposingOrDisposed()) return Promise.resolve(); - this.splatRenderReady = false; let splatProcessingTaskId = null; - const removeSplatProcessingTask = () => { if (splatProcessingTaskId !== null) { this.loadingSpinner.removeTask(splatProcessingTaskId); @@ -1062,45 +1097,7 @@ export class Viewer { } }; - const finish = (buildResults, resolver) => { - if (this.isDisposingOrDisposed()) return; - - // If we aren't calculating the splat distances from the center on the GPU, the sorting worker needs splat centers and - // transform indexes so that it can calculate those distance values. - if (!this.gpuAcceleratedSort && this.sortWorker) { - this.sortWorker.postMessage({ - 'centers': buildResults.centers.buffer, - 'sceneIndexes': buildResults.sceneIndexes.buffer, - 'range': { - 'from': buildResults.from, - 'to': buildResults.to, - 'count': buildResults.count - } - }); - } - - this.runSplatSort(true).then((sortRunning) => { - if (!this.sortWorker || !sortRunning) { - this.splatRenderReady = true; - removeSplatProcessingTask(); - resolver(); - } else { - if (enableRenderBeforeFirstSort) { - this.splatRenderReady = true; - } else { - this.runAfterNextSort.push(() => { - this.splatRenderReady = true; - }); - } - this.runAfterNextSort.push(() => { - removeSplatProcessingTask(); - resolver(); - }); - } - }); - - }; - + this.splatRenderReady = false; return new Promise((resolve) => { if (showLoadingUI) { splatProcessingTaskId = this.loadingSpinner.addTask('Processing splats...'); @@ -1112,12 +1109,45 @@ export class Viewer { const buildResults = this.addSplatBuffersToMesh(splatBuffers, splatBufferOptions, finalBuild, showLoadingUIForSplatTreeBuild, replaceExisting, preserveVisibleRegion); + const maxSplatCount = this.splatMesh.getMaxSplatCount(); if (this.sortWorker && this.sortWorker.maxSplatCount !== maxSplatCount) this.disposeSortWorker(); + // If we aren't calculating the splat distances from the center on the GPU, the sorting worker needs + // splat centers and transform indexes so that it can calculate those distance values. + if (!this.gpuAcceleratedSort) { + this.preSortMessages.push({ + 'centers': buildResults.centers.buffer, + 'sceneIndexes': buildResults.sceneIndexes.buffer, + 'range': { + 'from': buildResults.from, + 'to': buildResults.to, + 'count': buildResults.count + } + }); + } const sortWorkerSetupPromise = (!this.sortWorker && maxSplatCount > 0) ? this.setupSortWorker(this.splatMesh) : Promise.resolve(); sortWorkerSetupPromise.then(() => { - finish(buildResults, resolve); + if (this.isDisposingOrDisposed()) return; + this.runSplatSort(true, true).then((sortRunning) => { + if (!this.sortWorker || !sortRunning) { + this.splatRenderReady = true; + removeSplatProcessingTask(); + resolve(); + } else { + if (enableRenderBeforeFirstSort) { + this.splatRenderReady = true; + } else { + this.runAfterNextSort.push(() => { + this.splatRenderReady = true; + }); + } + this.runAfterNextSort.push(() => { + removeSplatProcessingTask(); + resolve(); + }); + } + }); }); } }, true); @@ -1168,7 +1198,7 @@ export class Viewer { if (showLoadingUIForSplatTreeBuild && splatCount >= MIN_SPLAT_COUNT_TO_SHOW_SPLAT_TREE_LOADING_SPINNER) { if (!finished && !splatOptimizingTaskId) { this.loadingSpinner.setMinimized(true, true); - splatOptimizingTaskId = this.loadingSpinner.addTask('Optimizing splats...'); + splatOptimizingTaskId = this.loadingSpinner.addTask('Optimizing data structures...'); } } }; @@ -1199,7 +1229,7 @@ export class Viewer { const splatCount = splatMesh.getSplatCount(); const maxSplatCount = splatMesh.getMaxSplatCount(); this.sortWorker = createSortWorker(maxSplatCount, this.sharedMemoryForWorkers, this.enableSIMDInSort, - this.integerBasedSort, this.splatMesh.dynamicMode); + this.integerBasedSort, this.splatMesh.dynamicMode, this.splatSortDistanceMapPrecision); this.sortWorker.onmessage = (e) => { if (e.data.sortDone) { this.sortRunning = false; @@ -1209,6 +1239,9 @@ export class Viewer { const sortedIndexes = new Uint32Array(e.data.sortedIndexes.buffer, 0, e.data.splatRenderCount); this.splatMesh.updateRenderIndexes(sortedIndexes, e.data.splatRenderCount); } + + this.lastSplatSortCount = this.splatSortCount; + this.lastSortTime = e.data.sortTime; this.sortPromiseResolver(); this.sortPromiseResolver = null; @@ -1264,6 +1297,7 @@ export class Viewer { this.sortPromiseResolver(); this.sortPromiseResolver = null; } + this.preSortMessages = []; this.sortRunning = false; } @@ -1397,7 +1431,9 @@ export class Viewer { */ stop() { if (this.selfDrivenMode && this.selfDrivenModeRunning) { - if (!this.webXRMode) { + if (this.webXRMode) { + this.renderer.setAnimationLoop(null); + } else { cancelAnimationFrame(this.requestFrameId); } this.selfDrivenModeRunning = false; @@ -1426,10 +1462,15 @@ export class Viewer { this.disposing = true; this.disposePromise = Promise.all(waitPromises).finally(() => { this.stop(); - if (this.controls) { - this.controls.dispose(); - this.controls = null; + if (this.orthographicControls) { + this.orthographicControls.dispose(); + this.orthographicControls = null; + } + if (this.perspectiveControls) { + this.perspectiveControls.dispose(); + this.perspectiveControls = null; } + this.controls = null; if (this.splatMesh) { this.splatMesh.dispose(); this.splatMesh = null; @@ -1507,6 +1548,8 @@ export class Viewer { const changeEpsilon = 0.0001; return function() { + if (!this.initialized || !this.splatRenderReady || this.isDisposingOrDisposed()) return false; + let shouldRender = false; let cameraChanged = false; if (this.camera) { @@ -1538,7 +1581,7 @@ export class Viewer { render = function() { return function() { - if (!this.initialized || !this.splatRenderReady) return; + if (!this.initialized || !this.splatRenderReady || this.isDisposingOrDisposed()) return; const hasRenderables = (threeScene) => { for (let child of threeScene.children) { @@ -1563,7 +1606,9 @@ export class Viewer { update(renderer, camera) { if (this.dropInMode) this.updateForDropInMode(renderer, camera); - if (!this.initialized || !this.splatRenderReady) return; + + if (!this.initialized || !this.splatRenderReady || this.isDisposingOrDisposed()) return; + if (this.controls) { this.controls.update(); if (this.camera.isOrthographicCamera && !this.usingExternalCamera) { @@ -1792,12 +1837,12 @@ export class Viewer { } ]; - return function(force = false) { + return function(force = false, forceSortAll = false) { if (!this.initialized) return Promise.resolve(false); if (this.sortRunning) return Promise.resolve(true); if (this.splatMesh.getSplatCount() <= 0) { this.splatRenderCount = 0; - return false; + return Promise.resolve(false); } let angleDiff = 0; @@ -1818,7 +1863,8 @@ export class Viewer { } this.sortRunning = true; - const { splatRenderCount, shouldSortAll } = this.gatherSceneNodesForSort(); + let { splatRenderCount, shouldSortAll } = this.gatherSceneNodesForSort(); + shouldSortAll = shouldSortAll || forceSortAll; this.splatRenderCount = splatRenderCount; mvpMatrix.copy(this.camera.matrixWorld).invert(); @@ -1826,17 +1872,17 @@ export class Viewer { mvpMatrix.premultiply(mvpCamera.projectionMatrix); mvpMatrix.multiply(this.splatMesh.matrixWorld); - let gpuAcceleratedSortPromise = Promise.resolve(); + let gpuAcceleratedSortPromise = Promise.resolve(true); if (this.gpuAcceleratedSort && (queuedSorts.length <= 1 || queuedSorts.length % 2 === 0)) { gpuAcceleratedSortPromise = this.splatMesh.computeDistancesOnGPU(mvpMatrix, this.sortWorkerPrecomputedDistances); } gpuAcceleratedSortPromise.then(() => { - if (this.splatMesh.dynamicMode || shouldSortAll) { - queuedSorts.push(this.splatRenderCount); - } else { - if (queuedSorts.length === 0) { - for (let partialSort of partialSorts) { + if (queuedSorts.length === 0) { + if (this.splatMesh.dynamicMode || shouldSortAll) { + queuedSorts.push(this.splatRenderCount); + } else { + for (let partialSort of partialSorts) { if (angleDiff < partialSort.angleThreshold) { for (let sortFraction of partialSort.sortFractions) { queuedSorts.push(Math.floor(this.splatRenderCount * sortFraction)); @@ -1848,6 +1894,7 @@ export class Viewer { } } let sortCount = Math.min(queuedSorts.shift(), this.splatRenderCount); + this.splatSortCount = sortCount; cameraPositionArray[0] = this.camera.position.x; cameraPositionArray[1] = this.camera.position.y; @@ -1875,6 +1922,12 @@ export class Viewer { this.sortPromiseResolver = resolve; }); + if (this.preSortMessages.length > 0) { + this.preSortMessages.forEach((message) => { + this.sortWorker.postMessage(message); + }); + this.preSortMessages = []; + } this.sortWorker.postMessage({ 'sort': sortMessage }); @@ -1883,6 +1936,8 @@ export class Viewer { lastSortViewPos.copy(this.camera.position); lastSortViewDir.copy(sortViewDir); } + + return true; }); return gpuAcceleratedSortPromise; diff --git a/src/loaders/DirectLoadError.js b/src/loaders/DirectLoadError.js new file mode 100644 index 00000000..1672d595 --- /dev/null +++ b/src/loaders/DirectLoadError.js @@ -0,0 +1,7 @@ +export class DirectLoadError extends Error { + + constructor(msg) { + super(msg); + } + +} diff --git a/src/loaders/InternalLoadType.js b/src/loaders/InternalLoadType.js new file mode 100644 index 00000000..f3e37e18 --- /dev/null +++ b/src/loaders/InternalLoadType.js @@ -0,0 +1,5 @@ +export const InternalLoadType = { + DirectToSplatBuffer: 0, + DirectToSplatArray: 1, + DownloadBeforeProcessing: 2 +}; diff --git a/src/loaders/SplatPartitioner.js b/src/loaders/SplatPartitioner.js index 36955796..f9c38eb5 100644 --- a/src/loaders/SplatPartitioner.js +++ b/src/loaders/SplatPartitioner.js @@ -32,7 +32,7 @@ export class SplatPartitioner { const sectionFilter = sectionFilters[s]; for (let i = 0; i < splatArray.splatCount; i++) { if (sectionFilter(i)) { - sectionSplats.addSplatFromArray(splatArray, i); + sectionSplats.addSplat(splatArray.splats[i]); } } newArrays.push(sectionSplats); diff --git a/src/loaders/ksplat/KSplatLoader.js b/src/loaders/ksplat/KSplatLoader.js index 8520a382..95973cab 100644 --- a/src/loaders/ksplat/KSplatLoader.js +++ b/src/loaders/ksplat/KSplatLoader.js @@ -19,9 +19,9 @@ export class KSplatLoader { } }; - static loadFromURL(fileName, externalOnProgress, progressiveLoad, onSectionBuilt) { - let progressiveLoadBuffer; - let progressiveLoadSplatBuffer; + static loadFromURL(fileName, externalOnProgress, loadDirectoToSplatBuffer, onSectionBuilt) { + let directLoadBuffer; + let directLoadSplatBuffer; let headerBuffer; let header; @@ -43,7 +43,7 @@ export class KSplatLoader { let chunks = []; - const progressiveLoadPromise = nativePromiseWithExtractedComponents(); + const directLoadPromise = nativePromiseWithExtractedComponents(); const checkAndLoadHeader = () => { if (!headerLoaded && !headerLoading && numBytesLoaded >= SplatBuffer.HeaderSizeBytes) { @@ -91,12 +91,12 @@ export class KSplatLoader { } const totalStorageSizeBytes = SplatBuffer.HeaderSizeBytes + header.maxSectionCount * SplatBuffer.SectionHeaderSizeBytes + totalSectionStorageStorageByes; - if (!progressiveLoadBuffer) { - progressiveLoadBuffer = new ArrayBuffer(totalStorageSizeBytes); + if (!directLoadBuffer) { + directLoadBuffer = new ArrayBuffer(totalStorageSizeBytes); let offset = 0; for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; - new Uint8Array(progressiveLoadBuffer, offset, chunk.byteLength).set(new Uint8Array(chunk)); + new Uint8Array(directLoadBuffer, offset, chunk.byteLength).set(new Uint8Array(chunk)); offset += chunk.byteLength; } } @@ -133,7 +133,7 @@ export class KSplatLoader { numBytesProgressivelyLoaded += Constants.ProgressiveLoadSectionSize; loadComplete = numBytesProgressivelyLoaded >= totalBytesToDownload; - if (!progressiveLoadSplatBuffer) progressiveLoadSplatBuffer = new SplatBuffer(progressiveLoadBuffer, false); + if (!directLoadSplatBuffer) directLoadSplatBuffer = new SplatBuffer(directLoadBuffer, false); const baseDataOffset = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes * header.maxSectionCount; let sectionBase = 0; @@ -153,15 +153,15 @@ export class KSplatLoader { let loadedSplatsForSection = Math.floor(bytesPastSSectionSplatDataStart / bytesPerSplat); loadedSplatsForSection = Math.min(loadedSplatsForSection, sectionHeader.maxSplatCount); loadedSplatCount += loadedSplatsForSection; - progressiveLoadSplatBuffer.updateLoadedCounts(reachedSections, loadedSplatCount); - progressiveLoadSplatBuffer.updateSectionLoadedCounts(i, loadedSplatsForSection); + directLoadSplatBuffer.updateLoadedCounts(reachedSections, loadedSplatCount); + directLoadSplatBuffer.updateSectionLoadedCounts(i, loadedSplatsForSection); } else { break; } sectionBase += sectionHeader.storageSizeBytes; } - onSectionBuilt(progressiveLoadSplatBuffer, loadComplete); + onSectionBuilt(directLoadSplatBuffer, loadComplete); const percentComplete = numBytesProgressivelyLoaded / totalBytesToDownload * 100; const percentLabel = (percentComplete).toFixed(2) + '%'; @@ -169,7 +169,7 @@ export class KSplatLoader { if (externalOnProgress) externalOnProgress(percentComplete, percentLabel, LoaderStatus.Downloading); if (loadComplete) { - progressiveLoadPromise.resolve(progressiveLoadSplatBuffer); + directLoadPromise.resolve(directLoadSplatBuffer); } else { checkAndLoadSections(); } @@ -182,12 +182,12 @@ export class KSplatLoader { const localOnProgress = (percent, percentStr, chunk) => { if (chunk) { chunks.push(chunk); - if (progressiveLoadBuffer) { - new Uint8Array(progressiveLoadBuffer, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); + if (directLoadBuffer) { + new Uint8Array(directLoadBuffer, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); } numBytesLoaded += chunk.byteLength; } - if (progressiveLoad) { + if (loadDirectoToSplatBuffer) { checkAndLoadHeader(); checkAndLoadSectionHeaders(); checkAndLoadSections(); @@ -196,9 +196,9 @@ export class KSplatLoader { } }; - return fetchWithProgress(fileName, localOnProgress, !progressiveLoad).then((fullBuffer) => { + return fetchWithProgress(fileName, localOnProgress, !loadDirectoToSplatBuffer).then((fullBuffer) => { if (externalOnProgress) externalOnProgress(0, '0%', LoaderStatus.Processing); - const loadPromise = progressiveLoad ? progressiveLoadPromise.promise : KSplatLoader.loadFromFileData(fullBuffer); + const loadPromise = loadDirectoToSplatBuffer ? directLoadPromise.promise : KSplatLoader.loadFromFileData(fullBuffer); return loadPromise.then((splatBuffer) => { if (externalOnProgress) externalOnProgress(100, '100%', LoaderStatus.Done); return splatBuffer; diff --git a/src/loaders/ply/INRIAV1PlyParser.js b/src/loaders/ply/INRIAV1PlyParser.js index 4fbaed82..1351bdf6 100644 --- a/src/loaders/ply/INRIAV1PlyParser.js +++ b/src/loaders/ply/INRIAV1PlyParser.js @@ -82,6 +82,16 @@ export class INRIAV1PlyParser { } } + parseToUncompressedSplatArraySection(header, fromSplat, toSplat, splatData, splatDataOffset, + splatArray, outSphericalHarmonicsDegree = 0) { + outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); + for (let i = fromSplat; i <= toSplat; i++) { + const parsedSplat = INRIAV1PlyParser.parseToUncompressedSplat(splatData, i, header, + splatDataOffset, outSphericalHarmonicsDegree); + splatArray.addSplat(parsedSplat); + } + } + decodeSectionSplatData(sectionSplatData, splatCount, sectionHeader, outSphericalHarmonicsDegree) { outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, sectionHeader.sphericalHarmonicsDegree); const splatArray = new UncompressedSplatArray(outSphericalHarmonicsDegree); diff --git a/src/loaders/ply/PlayCanvasCompressedPlyParser.js b/src/loaders/ply/PlayCanvasCompressedPlyParser.js index 17cde87b..cd57eb92 100644 --- a/src/loaders/ply/PlayCanvasCompressedPlyParser.js +++ b/src/loaders/ply/PlayCanvasCompressedPlyParser.js @@ -378,6 +378,22 @@ export class PlayCanvasCompressedPlyParser { } } + static parseToUncompressedSplatArraySection(chunkElement, vertexElement, fromIndex, toIndex, chunkSplatIndexOffset, + vertexDataBuffer, veretxReadOffset, splatArray, propertyFilter = null) { + + PlayCanvasCompressedPlyParser.readElementData(vertexElement, vertexDataBuffer, veretxReadOffset, fromIndex, toIndex, propertyFilter); + + const { positionExtremes, scaleExtremes, position, rotation, scale, color } = + PlayCanvasCompressedPlyParser.getElementStorageArrays(chunkElement, vertexElement); + + for (let i = fromIndex; i <= toIndex; ++i) { + const tempSplat = UncompressedSplatArray.createSplat(); + PlayCanvasCompressedPlyParser.decompressSplat(i, chunkSplatIndexOffset, position, positionExtremes, + scale, scaleExtremes, rotation, color, tempSplat); + splatArray.addSplat(tempSplat); + } + } + static parseToUncompressedSplatArray(plyBuffer) { const { chunkElement, vertexElement } = PlayCanvasCompressedPlyParser.readPly(plyBuffer); diff --git a/src/loaders/ply/PlyLoader.js b/src/loaders/ply/PlyLoader.js index 8ad46c54..4dabc9bf 100644 --- a/src/loaders/ply/PlyLoader.js +++ b/src/loaders/ply/PlyLoader.js @@ -8,7 +8,10 @@ import { fetchWithProgress, delayedExecute, nativePromiseWithExtractedComponents import { SplatBuffer } from '../SplatBuffer.js'; import { SplatBufferGenerator } from '../SplatBufferGenerator.js'; import { LoaderStatus } from '../LoaderStatus.js'; +import { DirectLoadError } from '../DirectLoadError.js'; import { Constants } from '../../Constants.js'; +import { UncompressedSplatArray } from '../UncompressedSplatArray.js'; +import { InternalLoadType } from '../InternalLoadType.js'; function storeChunksInBuffer(chunks, buffer) { let inBytes = 0; @@ -27,18 +30,32 @@ function storeChunksInBuffer(chunks, buffer) { return buffer; } +function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, sectionSize, sceneCenter, blockSize, bucketSize) { + if (optimizeSplatData) { + const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, + sectionSize, sceneCenter, + blockSize, bucketSize); + return splatBufferGenerator.generateFromUncompressedSplatArray(splatData); + } else { + return SplatBuffer.generateFromUncompressedSplatArrays([splatData], minimumAlpha, 0, new THREE.Vector3()); + } +} + export class PlyLoader { - static loadFromURL(fileName, onProgress, progressiveLoad, onStreamedSectionProgress, minimumAlpha, compressionLevel, - outSphericalHarmonicsDegree = 0, sectionSize, sceneCenter, blockSize, bucketSize) { + static loadFromURL(fileName, onProgress, loadDirectoToSplatBuffer, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, + optimizeSplatData = true, outSphericalHarmonicsDegree = 0, sectionSize, sceneCenter, blockSize, bucketSize) { + + let internalLoadType = loadDirectoToSplatBuffer ? InternalLoadType.DirectToSplatBuffer : InternalLoadType.DirectToSplatArray; + if (optimizeSplatData) internalLoadType = InternalLoadType.DirectToSplatArray; - const progressiveLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; + const directLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; const splatDataOffsetBytes = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes; const sectionCount = 1; - let progressiveLoadBufferIn; - let progressiveLoadBufferOut; - let progressiveLoadSplatBuffer; + let directLoadBufferIn; + let directLoadBufferOut; + let directLoadSplatBuffer; let compressedPlyHeaderChunksBuffer; let maxSplatCount = 0; let splatCount = 0; @@ -47,7 +64,7 @@ export class PlyLoader { let readyToLoadSplatData = false; let compressed = false; - const progressiveLoadPromise = nativePromiseWithExtractedComponents(); + const loadPromise = nativePromiseWithExtractedComponents(); let numBytesStreamed = 0; let numBytesParsed = 0; @@ -56,24 +73,29 @@ export class PlyLoader { let header = null; let chunks = []; - const textDecoder = new TextDecoder(); + let standardLoadUncompressedSplatArray; + const textDecoder = new TextDecoder(); const inriaV1PlyParser = new INRIAV1PlyParser(); const localOnProgress = (percent, percentLabel, chunkData) => { const loadComplete = percent >= 100; - if (progressiveLoad) { - - if (chunkData) { - chunks.push({ - 'data': chunkData, - 'sizeBytes': chunkData.byteLength, - 'startBytes': numBytesDownloaded, - 'endBytes': numBytesDownloaded + chunkData.byteLength - }); - numBytesDownloaded += chunkData.byteLength; - } + if (chunkData) { + chunks.push({ + 'data': chunkData, + 'sizeBytes': chunkData.byteLength, + 'startBytes': numBytesDownloaded, + 'endBytes': numBytesDownloaded + chunkData.byteLength + }); + numBytesDownloaded += chunkData.byteLength; + } + + if (internalLoadType === InternalLoadType.DownloadBeforeProcessing) { + if (loadComplete) { + loadPromise.resolve(chunks); + } + } else { if (!headerLoaded) { headerText += textDecoder.decode(chunkData); if (PlyParserUtils.checkTextForEndHeader(headerText)) { @@ -88,23 +110,33 @@ export class PlyLoader { maxSplatCount = header.vertexElement.count; compressed = true; } else { - throw new Error('PlyLoader.loadFromURL() -> Selected Ply format cannot be progressively loaded.'); + if (loadDirectoToSplatBuffer) { + throw new DirectLoadError('PlyLoader.loadFromURL() -> Selected Ply format cannot be directly loaded.'); + } else { + internalLoadType = InternalLoadType.DownloadBeforeProcessing; + return; + } } outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; const splatBufferSizeBytes = splatDataOffsetBytes + shDescriptor.BytesPerSplat * maxSplatCount; - progressiveLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); - SplatBuffer.writeHeaderToBuffer({ - versionMajor: SplatBuffer.CurrentMajorVersion, - versionMinor: SplatBuffer.CurrentMinorVersion, - maxSectionCount: sectionCount, - sectionCount: sectionCount, - maxSplatCount: maxSplatCount, - splatCount: splatCount, - compressionLevel: 0, - sceneCenter: new THREE.Vector3() - }, progressiveLoadBufferOut); + + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + directLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); + SplatBuffer.writeHeaderToBuffer({ + versionMajor: SplatBuffer.CurrentMajorVersion, + versionMinor: SplatBuffer.CurrentMinorVersion, + maxSectionCount: sectionCount, + sectionCount: sectionCount, + maxSplatCount: maxSplatCount, + splatCount: splatCount, + compressionLevel: 0, + sceneCenter: new THREE.Vector3() + }, directLoadBufferOut); + } else { + standardLoadUncompressedSplatArray = new UncompressedSplatArray(outSphericalHarmonicsDegree); + } numBytesStreamed = header.headerSizeBytes; numBytesParsed = header.headerSizeBytes; @@ -115,7 +147,7 @@ export class PlyLoader { compressedPlyHeaderChunksBuffer = storeChunksInBuffer(chunks, compressedPlyHeaderChunksBuffer); if (compressedPlyHeaderChunksBuffer.byteLength >= sizeRequiredForHeaderAndChunks) { PlayCanvasCompressedPlyParser.readElementData(header.chunkElement, compressedPlyHeaderChunksBuffer, - header.headerSizeBytes); + header.headerSizeBytes); numBytesStreamed = sizeRequiredForHeaderAndChunks; numBytesParsed = sizeRequiredForHeaderAndChunks; readyToLoadSplatData = true; @@ -126,52 +158,72 @@ export class PlyLoader { if (chunks.length > 0) { - progressiveLoadBufferIn = storeChunksInBuffer(chunks, progressiveLoadBufferIn); + directLoadBufferIn = storeChunksInBuffer(chunks, directLoadBufferIn); const bytesLoadedSinceLastStreamedSection = numBytesDownloaded - numBytesStreamed; - if (bytesLoadedSinceLastStreamedSection > progressiveLoadSectionSizeBytes || loadComplete) { + if (bytesLoadedSinceLastStreamedSection > directLoadSectionSizeBytes || loadComplete) { const numBytesToProcess = numBytesDownloaded - numBytesParsed; const addedSplatCount = Math.floor(numBytesToProcess / header.bytesPerSplat); const numBytesToParse = addedSplatCount * header.bytesPerSplat; const numBytesLeftOver = numBytesToProcess - numBytesToParse; const newSplatCount = splatCount + addedSplatCount; const parsedDataViewOffset = numBytesParsed - chunks[0].startBytes; - const dataToParse = new DataView(progressiveLoadBufferIn, parsedDataViewOffset, numBytesToParse); + const dataToParse = new DataView(directLoadBufferIn, parsedDataViewOffset, numBytesToParse); const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; const outOffset = splatCount * shDescriptor.BytesPerSplat + splatDataOffsetBytes; - if (compressed) { - PlayCanvasCompressedPlyParser.parseToUncompressedSplatBufferSection(header.chunkElement, + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + if (compressed) { + PlayCanvasCompressedPlyParser.parseToUncompressedSplatBufferSection(header.chunkElement, + header.vertexElement, 0, + addedSplatCount - 1, splatCount, + dataToParse, 0, + directLoadBufferOut, outOffset); + } else { + inriaV1PlyParser.parseToUncompressedSplatBufferSection(header, 0, addedSplatCount - 1, dataToParse, + 0, directLoadBufferOut, outOffset, + outSphericalHarmonicsDegree); + } + } else { + if (compressed) { + PlayCanvasCompressedPlyParser.parseToUncompressedSplatArraySection(header.chunkElement, header.vertexElement, 0, addedSplatCount - 1, splatCount, dataToParse, 0, - progressiveLoadBufferOut, outOffset); - } else { - inriaV1PlyParser.parseToUncompressedSplatBufferSection(header, 0, addedSplatCount - 1, dataToParse, - 0, progressiveLoadBufferOut, outOffset, - outSphericalHarmonicsDegree); + standardLoadUncompressedSplatArray); + } else { + inriaV1PlyParser.parseToUncompressedSplatArraySection(header, 0, addedSplatCount - 1, dataToParse, + 0, standardLoadUncompressedSplatArray, + outSphericalHarmonicsDegree); + } } splatCount = newSplatCount; - if (!progressiveLoadSplatBuffer) { - SplatBuffer.writeSectionHeaderToBuffer({ - maxSplatCount: maxSplatCount, - splatCount: splatCount, - bucketSize: 0, - bucketCount: 0, - bucketBlockSize: 0, - compressionScaleRange: 0, - storageSizeBytes: 0, - fullBucketCount: 0, - partiallyFilledBucketCount: 0, - sphericalHarmonicsDegree: outSphericalHarmonicsDegree - }, 0, progressiveLoadBufferOut, SplatBuffer.HeaderSizeBytes); - progressiveLoadSplatBuffer = new SplatBuffer(progressiveLoadBufferOut, false); + + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + if (!directLoadSplatBuffer) { + SplatBuffer.writeSectionHeaderToBuffer({ + maxSplatCount: maxSplatCount, + splatCount: splatCount, + bucketSize: 0, + bucketCount: 0, + bucketBlockSize: 0, + compressionScaleRange: 0, + storageSizeBytes: 0, + fullBucketCount: 0, + partiallyFilledBucketCount: 0, + sphericalHarmonicsDegree: outSphericalHarmonicsDegree + }, 0, directLoadBufferOut, SplatBuffer.HeaderSizeBytes); + directLoadSplatBuffer = new SplatBuffer(directLoadBufferOut, false); + } + directLoadSplatBuffer.updateLoadedCounts(1, splatCount); + if (onProgressiveLoadSectionProgress) { + onProgressiveLoadSectionProgress(directLoadSplatBuffer, loadComplete); + } } - progressiveLoadSplatBuffer.updateLoadedCounts(1, splatCount); - onStreamedSectionProgress(progressiveLoadSplatBuffer, loadComplete); - numBytesStreamed += progressiveLoadSectionSizeBytes; + + numBytesStreamed += directLoadSectionSizeBytes; numBytesParsed += numBytesToParse; if (numBytesLeftOver === 0) { @@ -191,35 +243,49 @@ export class PlyLoader { } if (loadComplete) { - progressiveLoadPromise.resolve(progressiveLoadSplatBuffer); + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + loadPromise.resolve(directLoadSplatBuffer); + } else { + loadPromise.resolve(standardLoadUncompressedSplatArray); + } } } - } + if (onProgress) onProgress(percent, percentLabel, LoaderStatus.Downloading); }; - return fetchWithProgress(fileName, localOnProgress, !progressiveLoad).then((plyFileData) => { + if (onProgress) onProgress(0, '0%', LoaderStatus.Downloading); + return fetchWithProgress(fileName, localOnProgress, false).then(() => { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); - const loadPromise = progressiveLoad ? progressiveLoadPromise.promise : - PlyLoader.loadFromFileData(plyFileData, minimumAlpha, compressionLevel, outSphericalHarmonicsDegree, - sectionSize, sceneCenter, blockSize, bucketSize); - return loadPromise.then((splatBuffer) => { + return loadPromise.promise.then((splatData) => { if (onProgress) onProgress(100, '100%', LoaderStatus.Done); - return splatBuffer; + if (internalLoadType === InternalLoadType.DownloadBeforeProcessing) { + const chunkDatas = chunks.map((chunk) => chunk.data); + return new Blob(chunkDatas).arrayBuffer().then((plyFileData) => { + return PlyLoader.loadFromFileData(plyFileData, minimumAlpha, compressionLevel, optimizeSplatData, + outSphericalHarmonicsDegree, sectionSize, sceneCenter, blockSize, bucketSize); + }); + } else if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + return splatData; + } else { + return delayedExecute(() => { + return finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, + sectionSize, sceneCenter, blockSize, bucketSize); + }); + } }); }); } - static loadFromFileData(plyFileData, minimumAlpha, compressionLevel, outSphericalHarmonicsDegree = 0, + static loadFromFileData(plyFileData, minimumAlpha, compressionLevel, optimizeSplatData, outSphericalHarmonicsDegree = 0, sectionSize, sceneCenter, blockSize, bucketSize) { return delayedExecute(() => { return PlyParser.parseToUncompressedSplatArray(plyFileData, outSphericalHarmonicsDegree); }) .then((splatArray) => { - const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, sectionSize, - sceneCenter, blockSize, bucketSize); - return splatBufferGenerator.generateFromUncompressedSplatArray(splatArray); + return finalize(splatArray, optimizeSplatData, minimumAlpha, compressionLevel, + sectionSize, sceneCenter, blockSize, bucketSize); }); } } diff --git a/src/loaders/splat/SplatLoader.js b/src/loaders/splat/SplatLoader.js index 5dd067c9..59a535da 100644 --- a/src/loaders/splat/SplatLoader.js +++ b/src/loaders/splat/SplatLoader.js @@ -3,25 +3,44 @@ import { SplatBuffer } from '../SplatBuffer.js'; import { SplatBufferGenerator } from '../SplatBufferGenerator.js'; import { SplatParser } from './SplatParser.js'; import { fetchWithProgress, delayedExecute, nativePromiseWithExtractedComponents } from '../../Util.js'; +import { UncompressedSplatArray } from '../UncompressedSplatArray.js'; import { LoaderStatus } from '../LoaderStatus.js'; +import { DirectLoadError } from '../DirectLoadError.js'; import { Constants } from '../../Constants.js'; +import { InternalLoadType } from '../InternalLoadType.js'; + +function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, sectionSize, sceneCenter, blockSize, bucketSize) { + if (optimizeSplatData) { + const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, + sectionSize, sceneCenter, + blockSize, bucketSize); + return splatBufferGenerator.generateFromUncompressedSplatArray(splatData); + } else { + return SplatBuffer.generateFromUncompressedSplatArrays([splatData], minimumAlpha, 0, new THREE.Vector3()); + } +} export class SplatLoader { - static loadFromURL(fileName, onProgress, progressiveLoad, onStreamedSectionProgress, minimumAlpha, compressionLevel, - optimizeSplatData, sectionSize, sceneCenter, blockSize, bucketSize) { + static loadFromURL(fileName, onProgress, loadDirectoToSplatBuffer, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, + optimizeSplatData = true, sectionSize, sceneCenter, blockSize, bucketSize) { + + let internalLoadType = loadDirectoToSplatBuffer ? InternalLoadType.DirectToSplatBuffer : InternalLoadType.DirectToSplatArray; + if (optimizeSplatData) internalLoadType = InternalLoadType.DirectToSplatArray; const splatDataOffsetBytes = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes; - const progressiveLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; + const directLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; const sectionCount = 1; - let progressiveLoadBufferIn; - let progressiveLoadBufferOut; - let progressiveLoadSplatBuffer; + let directLoadBufferIn; + let directLoadBufferOut; + let directLoadSplatBuffer; let maxSplatCount = 0; let splatCount = 0; - const progressiveLoadPromise = nativePromiseWithExtractedComponents(); + let standardLoadUncompressedSplatArray; + + const loadPromise = nativePromiseWithExtractedComponents(); let numBytesStreamed = 0; let numBytesLoaded = 0; @@ -29,14 +48,35 @@ export class SplatLoader { const localOnProgress = (percent, percentStr, chunk, fileSize) => { const loadComplete = percent >= 100; - if (!fileSize) progressiveLoad = false; - if (progressiveLoad) { - if (!progressiveLoadBufferIn) { - maxSplatCount = fileSize / SplatParser.RowSizeBytes; - progressiveLoadBufferIn = new ArrayBuffer(fileSize); - const bytesPerSplat = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[0].BytesPerSplat; - const splatBufferSizeBytes = splatDataOffsetBytes + bytesPerSplat * maxSplatCount; - progressiveLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); + + if (chunk) { + chunks.push(chunk); + } + + if (internalLoadType === InternalLoadType.DownloadBeforeProcessing) { + if (loadComplete) { + loadPromise.resolve(chunks); + } + return; + } + + if (!fileSize) { + if (loadDirectoToSplatBuffer) { + throw new DirectLoadError('Cannon directly load .splat because no file size info is available.'); + } else { + internalLoadType = InternalLoadType.DownloadBeforeProcessing; + return; + } + } + + if (!directLoadBufferIn) { + maxSplatCount = fileSize / SplatParser.RowSizeBytes; + directLoadBufferIn = new ArrayBuffer(fileSize); + const bytesPerSplat = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[0].BytesPerSplat; + const splatBufferSizeBytes = splatDataOffsetBytes + bytesPerSplat * maxSplatCount; + + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + directLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); SplatBuffer.writeHeaderToBuffer({ versionMajor: SplatBuffer.CurrentMajorVersion, versionMinor: SplatBuffer.CurrentMinorVersion, @@ -46,23 +86,34 @@ export class SplatLoader { splatCount: splatCount, compressionLevel: 0, sceneCenter: new THREE.Vector3() - }, progressiveLoadBufferOut); + }, directLoadBufferOut); + } else { + standardLoadUncompressedSplatArray = new UncompressedSplatArray(0); } + } + + if (chunk) { + new Uint8Array(directLoadBufferIn, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); + numBytesLoaded += chunk.byteLength; + + const bytesLoadedSinceLastSection = numBytesLoaded - numBytesStreamed; + if (bytesLoadedSinceLastSection > directLoadSectionSizeBytes || loadComplete) { + const bytesToUpdate = loadComplete ? bytesLoadedSinceLastSection : directLoadSectionSizeBytes; + const addedSplatCount = bytesToUpdate / SplatParser.RowSizeBytes; + const newSplatCount = splatCount + addedSplatCount; - if (chunk) { - chunks.push(chunk); - new Uint8Array(progressiveLoadBufferIn, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); - numBytesLoaded += chunk.byteLength; - - const bytesLoadedSinceLastSection = numBytesLoaded - numBytesStreamed; - if (bytesLoadedSinceLastSection > progressiveLoadSectionSizeBytes || loadComplete) { - const bytesToUpdate = loadComplete ? bytesLoadedSinceLastSection : progressiveLoadSectionSizeBytes; - const addedSplatCount = bytesToUpdate / SplatParser.RowSizeBytes; - const newSplatCount = splatCount + addedSplatCount; - SplatParser.parseToUncompressedSplatBufferSection(splatCount, newSplatCount - 1, progressiveLoadBufferIn, 0, - progressiveLoadBufferOut, splatDataOffsetBytes); - splatCount = newSplatCount; - if (!progressiveLoadSplatBuffer) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + SplatParser.parseToUncompressedSplatBufferSection(splatCount, newSplatCount - 1, directLoadBufferIn, 0, + directLoadBufferOut, splatDataOffsetBytes); + } else { + SplatParser.parseToUncompressedSplatArraySection(splatCount, newSplatCount - 1, directLoadBufferIn, 0, + standardLoadUncompressedSplatArray); + } + + splatCount = newSplatCount; + + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + if (!directLoadSplatBuffer) { SplatBuffer.writeSectionHeaderToBuffer({ maxSplatCount: maxSplatCount, splatCount: splatCount, @@ -73,30 +124,48 @@ export class SplatLoader { storageSizeBytes: 0, fullBucketCount: 0, partiallyFilledBucketCount: 0 - }, 0, progressiveLoadBufferOut, SplatBuffer.HeaderSizeBytes); - progressiveLoadSplatBuffer = new SplatBuffer(progressiveLoadBufferOut, false); + }, 0, directLoadBufferOut, SplatBuffer.HeaderSizeBytes); + directLoadSplatBuffer = new SplatBuffer(directLoadBufferOut, false); + } + directLoadSplatBuffer.updateLoadedCounts(1, splatCount); + if (onProgressiveLoadSectionProgress) { + onProgressiveLoadSectionProgress(directLoadSplatBuffer, loadComplete); } - progressiveLoadSplatBuffer.updateLoadedCounts(1, splatCount); - onStreamedSectionProgress(progressiveLoadSplatBuffer, loadComplete); - numBytesStreamed += progressiveLoadSectionSizeBytes; } + + numBytesStreamed += directLoadSectionSizeBytes; } - if (loadComplete) { - progressiveLoadPromise.resolve(progressiveLoadSplatBuffer); + } + + if (loadComplete) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + loadPromise.resolve(directLoadSplatBuffer); + } else { + loadPromise.resolve(standardLoadUncompressedSplatArray); } } + if (onProgress) onProgress(percent, percentStr, LoaderStatus.Downloading); - return progressiveLoad; }; - return fetchWithProgress(fileName, localOnProgress, true).then((fullBuffer) => { + if (onProgress) onProgress(0, '0%', LoaderStatus.Downloading); + return fetchWithProgress(fileName, localOnProgress, false).then(() => { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); - const loadPromise = progressiveLoad ? progressiveLoadPromise.promise : - SplatLoader.loadFromFileData(fullBuffer, minimumAlpha, compressionLevel, optimizeSplatData, - sectionSize, sceneCenter, blockSize, bucketSize); - return loadPromise.then((splatBuffer) => { + return loadPromise.promise.then((splatData) => { if (onProgress) onProgress(100, '100%', LoaderStatus.Done); - return splatBuffer; + if (internalLoadType === InternalLoadType.DownloadBeforeProcessing) { + return new Blob(chunks).arrayBuffer().then((splatData) => { + return SplatLoader.loadFromFileData(splatData, minimumAlpha, compressionLevel, optimizeSplatData, + sectionSize, sceneCenter, blockSize, bucketSize); + }); + } else if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + return splatData; + } else { + return delayedExecute(() => { + return finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, + sectionSize, sceneCenter, blockSize, bucketSize); + }); + } }); }); } @@ -105,14 +174,8 @@ export class SplatLoader { sectionSize, sceneCenter, blockSize, bucketSize) { return delayedExecute(() => { const splatArray = SplatParser.parseStandardSplatToUncompressedSplatArray(splatFileData); - if (optimizeSplatData) { - const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, - sectionSize, sceneCenter, blockSize, - bucketSize); - return splatBufferGenerator.generateFromUncompressedSplatArray(splatArray); - } else { - return SplatBuffer.generateFromUncompressedSplatArrays([splatArray], minimumAlpha, 0, new THREE.Vector3()); - } + return finalize(splatArray, optimizeSplatData, minimumAlpha, compressionLevel, + sectionSize, sceneCenter, blockSize, bucketSize); }); } diff --git a/src/loaders/splat/SplatParser.js b/src/loaders/splat/SplatParser.js index 444779ae..53c1c152 100644 --- a/src/loaders/splat/SplatParser.js +++ b/src/loaders/splat/SplatParser.js @@ -55,6 +55,25 @@ export class SplatParser { } } + static parseToUncompressedSplatArraySection(fromSplat, toSplat, fromBuffer, fromOffset, splatArray) { + + for (let i = fromSplat; i <= toSplat; i++) { + const inBase = i * SplatParser.RowSizeBytes + fromOffset; + const inCenter = new Float32Array(fromBuffer, inBase, 3); + const inScale = new Float32Array(fromBuffer, inBase + SplatParser.CenterSizeBytes, 3); + const inColor = new Uint8Array(fromBuffer, inBase + SplatParser.CenterSizeBytes + SplatParser.ScaleSizeBytes, 4); + const inRotation = new Uint8Array(fromBuffer, inBase + SplatParser.CenterSizeBytes + SplatParser.ScaleSizeBytes + + SplatParser.RotationSizeBytes, 4); + + const quat = new THREE.Quaternion((inRotation[1] - 128) / 128, (inRotation[2] - 128) / 128, + (inRotation[3] - 128) / 128, (inRotation[0] - 128) / 128); + quat.normalize(); + + splatArray.addSplatFromComonents(inCenter[0], inCenter[1], inCenter[2], inScale[0], inScale[1], inScale[2], + quat.w, quat.x, quat.y, quat.z, inColor[0], inColor[1], inColor[2], inColor[3]); + } + } + static parseStandardSplatToUncompressedSplatArray(inBuffer) { // Standard .splat row layout: // XYZ - Position (Float32) diff --git a/src/splatmesh/SplatMesh.js b/src/splatmesh/SplatMesh.js index f44d3ac8..6251efd1 100644 --- a/src/splatmesh/SplatMesh.js +++ b/src/splatmesh/SplatMesh.js @@ -49,7 +49,7 @@ export class SplatMesh extends THREE.Mesh { constructor(splatRenderMode = SplatRenderMode.ThreeD, dynamicMode = false, enableOptionalEffects = false, halfPrecisionCovariancesOnGPU = false, devicePixelRatio = 1, enableDistancesComputationOnGPU = true, integerBasedDistancesComputation = false, antialiased = false, maxScreenSpaceSplatSize = 1024, logLevel = LogLevel.None, - sphericalHarmonicsDegree = 0) { + sphericalHarmonicsDegree = 0, sceneFadeInRateMultiplier = 1.0) { super(dummyGeometry, dummyMaterial); // Reference to a Three.js renderer @@ -98,6 +98,8 @@ export class SplatMesh extends THREE.Mesh { this.sphericalHarmonicsDegree = sphericalHarmonicsDegree; this.minSphericalHarmonicsDegree = 0; + this.sceneFadeInRateMultiplier = sceneFadeInRateMultiplier; + // The individual splat scenes stored in this splat mesh, each containing their own transform this.scenes = []; @@ -373,14 +375,14 @@ export class SplatMesh extends THREE.Mesh { this.globalSplatIndexToSceneIndexMap = indexMaps.sceneIndexMap; } - const splatCount = this.getSplatCount(); + const splatBufferSplatCount = this.getSplatCount(true); if (this.enableDistancesComputationOnGPU) this.setupDistancesComputationTransformFeedback(); const dataUpdateResults = this.refreshGPUDataFromSplatBuffers(isUpdateBuild); for (let i = 0; i < this.scenes.length; i++) { this.lastBuildScenes[i] = this.scenes[i]; } - this.lastBuildSplatCount = splatCount; + this.lastBuildSplatCount = splatBufferSplatCount; this.lastBuildMaxSplatCount = this.getMaxSplatCount(); this.lastBuildSceneCount = this.scenes.length; @@ -580,7 +582,7 @@ export class SplatMesh extends THREE.Mesh { * @return {object} */ refreshGPUDataFromSplatBuffers(sinceLastBuildOnly) { - const splatCount = this.getSplatCount(); + const splatCount = this.getSplatCount(true); this.refreshDataTexturesFromSplatBuffers(sinceLastBuildOnly); const updateStart = sinceLastBuildOnly ? this.lastBuildSplatCount : 0; const { centers, sceneIndexes } = this.getDataForDistancesComputation(updateStart, splatCount - 1); @@ -613,7 +615,7 @@ export class SplatMesh extends THREE.Mesh { * @param {boolean} sinceLastBuildOnly Specify whether or not to only update for splats that have been added since the last build. */ refreshDataTexturesFromSplatBuffers(sinceLastBuildOnly) { - const splatCount = this.getSplatCount(); + const splatCount = this.getSplatCount(true); const fromSplat = this.lastBuildSplatCount; const toSplat = splatCount - 1; @@ -630,7 +632,7 @@ export class SplatMesh extends THREE.Mesh { setupDataTextures() { const maxSplatCount = this.getMaxSplatCount(); - const splatCount = this.getSplatCount(); + const splatCount = this.getSplatCount(true); this.disposeTextures(); @@ -1164,7 +1166,7 @@ export class SplatMesh extends THREE.Mesh { } updateVisibleRegion(sinceLastBuildOnly) { - const splatCount = this.getSplatCount(); + const splatCount = this.getSplatCount(true); const tempCenter = new THREE.Vector3(); if (!sinceLastBuildOnly) { const avgCenter = new THREE.Vector3(); @@ -1193,8 +1195,8 @@ export class SplatMesh extends THREE.Mesh { } updateVisibleRegionFadeDistance(sceneRevealMode = SceneRevealMode.Default) { - const fastFadeRate = SCENE_FADEIN_RATE_FAST; - const gradualFadeRate = SCENE_FADEIN_RATE_GRADUAL; + const fastFadeRate = SCENE_FADEIN_RATE_FAST * this.sceneFadeInRateMultiplier; + const gradualFadeRate = SCENE_FADEIN_RATE_GRADUAL * this.sceneFadeInRateMultiplier; const defaultFadeInRate = this.finalBuild ? fastFadeRate : gradualFadeRate; const fadeInRate = sceneRevealMode === SceneRevealMode.Default ? defaultFadeInRate : gradualFadeRate; this.visibleRegionFadeStartRadius = (this.visibleRegionRadius - this.visibleRegionFadeStartRadius) * @@ -1225,6 +1227,7 @@ export class SplatMesh extends THREE.Mesh { geometry.attributes.splatIndex.needsUpdate = true; if (renderSplatCount > 0 && this.firstRenderTime === -1) this.firstRenderTime = performance.now(); geometry.instanceCount = renderSplatCount; + geometry.setDrawRange(0, renderSplatCount); } /** @@ -1296,8 +1299,9 @@ export class SplatMesh extends THREE.Mesh { return this.splatDataTextures; } - getSplatCount() { - return SplatMesh.getTotalSplatCountForScenes(this.scenes); + getSplatCount(includeSinceLastBuild = false) { + if (!includeSinceLastBuild) return this.lastBuildSplatCount; + else return SplatMesh.getTotalSplatCountForScenes(this.scenes); } static getTotalSplatCountForScenes(scenes) { @@ -1844,7 +1848,7 @@ export class SplatMesh extends THREE.Mesh { */ fillSplatDataArrays(covariances, scales, rotations, centers, colors, sphericalHarmonics, applySceneTransform, covarianceCompressionLevel = 0, scaleRotationCompressionLevel = 0, sphericalHarmonicsCompressionLevel = 1, - srcStart, srcEnd, destStart = 0) { + srcStart, srcEnd, destStart = 0, sceneIndex) { const scaleOverride = new THREE.Vector3(); scaleOverride.x = undefined; scaleOverride.y = undefined; @@ -1855,7 +1859,13 @@ export class SplatMesh extends THREE.Mesh { } const tempTransform = new THREE.Matrix4(); - for (let i = 0; i < this.scenes.length; i++) { + let startSceneIndex = 0; + let endSceneIndex = this.scenes.length - 1; + if (sceneIndex !== undefined && sceneIndex !== null && sceneIndex >= 0 && sceneIndex <= this.scenes.length) { + startSceneIndex = sceneIndex; + endSceneIndex = sceneIndex; + } + for (let i = startSceneIndex; i <= endSceneIndex; i++) { if (applySceneTransform === undefined || applySceneTransform === null) { applySceneTransform = this.dynamicMode ? false : true; } @@ -2048,4 +2058,35 @@ export class SplatMesh extends THREE.Mesh { } return intMatrixArray; } + + computeBoundingBox(applySceneTransforms = false, sceneIndex) { + let splatCount = this.getSplatCount(); + if (sceneIndex !== undefined && sceneIndex !== null) { + if (sceneIndex < 0 || sceneIndex >= this.scenes.length) { + throw new Error('SplatMesh::computeBoundingBox() -> Invalid scene index.'); + } + splatCount = this.scenes[sceneIndex].splatBuffer.getSplatCount(); + } + + const floatCenters = new Float32Array(splatCount * 3); + this.fillSplatDataArrays(null, null, null, floatCenters, null, null, applySceneTransforms, + undefined, undefined, undefined, undefined, sceneIndex); + + const min = new THREE.Vector3(); + const max = new THREE.Vector3(); + for (let i = 0; i < splatCount; i++) { + const offset = i * 3; + const x = floatCenters[offset]; + const y = floatCenters[offset + 1]; + const z = floatCenters[offset + 2]; + if (i === 0 || x < min.x) min.x = x; + if (i === 0 || y < min.y) min.y = y; + if (i === 0 || z < min.z) min.z = z; + if (i === 0 || x > max.x) max.x = x; + if (i === 0 || y > max.y) max.y = y; + if (i === 0 || z > max.z) max.z = z; + } + + return new THREE.Box3(min, max); + } } diff --git a/src/webxr/VRButton.js b/src/webxr/VRButton.js index 08208d75..5732eb92 100644 --- a/src/webxr/VRButton.js +++ b/src/webxr/VRButton.js @@ -14,7 +14,7 @@ all copies or substantial portions of the Software. export class VRButton { - static createButton( renderer ) { + static createButton( renderer, sessionInit = {} ) { const button = document.createElement( 'button' ); @@ -60,7 +60,15 @@ export class VRButton { // ('local' is always available for immersive sessions and doesn't need to // be requested separately.) - const sessionInit = { optionalFeatures: ['local-floor', 'bounded-floor', 'hand-tracking', 'layers'] }; + const sessionOptions = { + ...sessionInit, + optionalFeatures: [ + 'local-floor', + 'bounded-floor', + 'layers', + ...( sessionInit.optionalFeatures || [] ) + ], + }; button.onmouseenter = function() { @@ -78,7 +86,7 @@ export class VRButton { if ( currentSession === null ) { - navigator.xr.requestSession( 'immersive-vr', sessionInit ).then( onSessionStarted ); + navigator.xr.requestSession( 'immersive-vr', sessionOptions ).then( onSessionStarted ); } else { @@ -86,7 +94,7 @@ export class VRButton { if ( navigator.xr.offerSession !== undefined ) { - navigator.xr.offerSession( 'immersive-vr', sessionInit ) + navigator.xr.offerSession( 'immersive-vr', sessionOptions ) .then( onSessionStarted ) .catch( ( err ) => { @@ -102,7 +110,7 @@ export class VRButton { if ( navigator.xr.offerSession !== undefined ) { - navigator.xr.offerSession( 'immersive-vr', sessionInit ) + navigator.xr.offerSession( 'immersive-vr', sessionOptions ) .then( onSessionStarted ) .catch( ( err ) => { diff --git a/src/worker/SortWorker.js b/src/worker/SortWorker.js index 9b73a0f1..077c5266 100644 --- a/src/worker/SortWorker.js +++ b/src/worker/SortWorker.js @@ -24,7 +24,8 @@ function sortWorker(self) { let modelViewProjOffset; let countsZero; let sortedIndexesOut; - + let distanceMapRange; + let uploadedSplatCount; let Constants; function sort(splatSortCount, splatRenderCount, modelViewProj, @@ -49,12 +50,12 @@ function sortWorker(self) { } } - if (!countsZero) countsZero = new Uint32Array(Constants.DepthMapRange); + if (!countsZero) countsZero = new Uint32Array(distanceMapRange); new Float32Array(wasmMemory, modelViewProjOffset, 16).set(modelViewProj); - new Uint32Array(wasmMemory, frequenciesOffset, Constants.DepthMapRange).set(countsZero); + new Uint32Array(wasmMemory, frequenciesOffset, distanceMapRange).set(countsZero); wasmInstance.exports.sortIndexes(indexesToSortOffset, centersOffset, precomputedDistancesOffset, mappedDistancesOffset, frequenciesOffset, modelViewProjOffset, - sortedIndexesOffset, sceneIndexesOffset, transformsOffset, Constants.DepthMapRange, + sortedIndexesOffset, sceneIndexesOffset, transformsOffset, distanceMapRange, splatSortCount, splatRenderCount, splatCount, usePrecomputedDistances, integerBasedSort, dynamicMode); @@ -94,12 +95,10 @@ function sortWorker(self) { new Uint32Array(wasmMemory, sceneIndexesOffset + e.data.range.from * 4, e.data.range.count).set(new Uint32Array(sceneIndexes)); } - self.postMessage({ - 'centerDataSet': true, - }); + uploadedSplatCount = e.data.range.from + e.data.range.count; } else if (e.data.sort) { - const renderCount = e.data.sort.splatRenderCount || 0; - const sortCount = e.data.sort.splatSortCount || 0; + const renderCount = Math.min(e.data.sort.splatRenderCount || 0, uploadedSplatCount); + const sortCount = Math.min(e.data.sort.splatSortCount || 0, uploadedSplatCount); const usePrecomputedDistances = e.data.sort.usePrecomputedDistances; let copyIndexesToSort; @@ -120,6 +119,8 @@ function sortWorker(self) { useSharedMemory = e.data.init.useSharedMemory; integerBasedSort = e.data.init.integerBasedSort; dynamicMode = e.data.init.dynamicMode; + distanceMapRange = e.data.init.distanceMapRange; + uploadedSplatCount = 0; const CENTERS_BYTES_PER_ENTRY = integerBasedSort ? (Constants.BytesPerInt * 4) : (Constants.BytesPerFloat * 4); @@ -133,7 +134,8 @@ function sortWorker(self) { (splatCount * Constants.BytesPerInt) : (splatCount * Constants.BytesPerFloat); const memoryRequiredForMappedDistances = splatCount * Constants.BytesPerInt; const memoryRequiredForSortedIndexes = splatCount * Constants.BytesPerInt; - const memoryRequiredForIntermediateSortBuffers = Constants.DepthMapRange * Constants.BytesPerInt * 2; + const memoryRequiredForIntermediateSortBuffers = integerBasedSort ? (distanceMapRange * Constants.BytesPerInt * 2) : + (distanceMapRange * Constants.BytesPerFloat * 2); const memoryRequiredforTransformIndexes = dynamicMode ? (splatCount * Constants.BytesPerInt) : 0; const memoryRequiredforTransforms = dynamicMode ? (Constants.MaxScenes * matrixSize) : 0; const extraMemory = Constants.MemoryPageSize * 32; @@ -197,7 +199,8 @@ function sortWorker(self) { }; } -export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, integerBasedSort, dynamicMode) { +export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, integerBasedSort, dynamicMode, + splatSortDistanceMapPrecision = Constants.DefaultSplatSortDistanceMapPrecision) { const worker = new Worker( URL.createObjectURL( new Blob(['(', sortWorker.toString(), ')(self)'], { @@ -209,16 +212,19 @@ export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, let sourceWasm = SorterWasm; // iOS makes choosing the right WebAssembly configuration tricky :( - let iOSSemVer = isIOS() ? getIOSSemever() : null; + const iOSSemVer = isIOS() ? getIOSSemever() : null; if (!enableSIMDInSort && !useSharedMemory) { sourceWasm = SorterWasmNoSIMD; - if (iOSSemVer && iOSSemVer.major < 16 && iOSSemVer.minor < 4) { + // Testing on various devices has shown that even when shared memory is disabled, the WASM module with shared + // memory can still be used most of the time -- the exception seems to be iOS devices below 16.4 + if (iOSSemVer && iOSSemVer.major <= 16 && iOSSemVer.minor < 4) { sourceWasm = SorterWasmNoSIMDNonShared; } } else if (!enableSIMDInSort) { sourceWasm = SorterWasmNoSIMD; } else if (!useSharedMemory) { - if (iOSSemVer && iOSSemVer.major < 16 && iOSSemVer.minor < 4) { + // Same issue with shared memory as above on iOS devices + if (iOSSemVer && iOSSemVer.major <= 16 && iOSSemVer.minor < 4) { sourceWasm = SorterWasmNonShared; } } @@ -236,11 +242,11 @@ export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, 'useSharedMemory': useSharedMemory, 'integerBasedSort': integerBasedSort, 'dynamicMode': dynamicMode, + 'distanceMapRange': 1 << splatSortDistanceMapPrecision, // Super hacky 'Constants': { 'BytesPerFloat': Constants.BytesPerFloat, 'BytesPerInt': Constants.BytesPerInt, - 'DepthMapRange': Constants.DepthMapRange, 'MemoryPageSize': Constants.MemoryPageSize, 'MaxScenes': Constants.MaxScenes }