From 358bffa52ad7de13aa26d3769337c8166da03549 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sun, 25 Aug 2024 19:54:48 -0700 Subject: [PATCH 01/25] Bug fixes --- src/ArrowHelper.js | 2 ++ src/Util.js | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/ArrowHelper.js b/src/ArrowHelper.js index ce309115..1ac163b4 100644 --- a/src/ArrowHelper.js +++ b/src/ArrowHelper.js @@ -1,5 +1,7 @@ import * as THREE from 'three'; +const _axis = new THREE.Vector3(); + export class ArrowHelper extends THREE.Object3D { constructor(dir = new THREE.Vector3(0, 0, 1), origin = new THREE.Vector3(0, 0, 0), length = 1, diff --git a/src/Util.js b/src/Util.js index 4e0a423b..5ab58f3d 100644 --- a/src/Util.js +++ b/src/Util.js @@ -67,6 +67,13 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { return new AbortablePromise((resolve, reject) => { fetch(path, { signal }) .then(async (data) => { + // Handle error conditions where data is still returned + if (!data.ok) { + const errorText = await data.text(); + reject(new Error(`Fetch failed: ${data.status} ${data.statusText} ${errorText}`)); + return; + } + const reader = data.body.getReader(); let bytesDownloaded = 0; let _fileSize = data.headers.get('Content-Length'); @@ -103,7 +110,7 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { } } catch (error) { reject(error); - break; + return; } } }) From b2e81566fddaed3d19bf627fb1fd9e5fb466548f Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Tue, 27 Aug 2024 20:00:54 -0700 Subject: [PATCH 02/25] Don't reject a resolved promise & vice-versa --- src/Viewer.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Viewer.js b/src/Viewer.js index 244cd914..2594af1f 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -863,7 +863,6 @@ export class Viewer { .then(() => { progressiveLoadedSectionBuilding = false; if (queuedBuild.firstBuild) { - progressiveLoadFirstSectionBuildPromise.reject = null; progressiveLoadFirstSectionBuildPromise.resolve(); } else if (queuedBuild.finalBuild) { splatSceneDownloadAndBuildPromise.resolve(); @@ -891,10 +890,10 @@ export class Viewer { } }; - let splatSceneDownloadPromise = this.downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold, onDownloadProgress, true, - onProgressiveLoadSectionProgress, format); + const splatSceneDownloadPromise = this.downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold, onDownloadProgress, true, + onProgressiveLoadSectionProgress, format); - const progressiveLoadFirstSectionBuildPromise = abortablePromiseWithExtractedComponents(splatSceneDownloadPromise.abortHandler); + let progressiveLoadFirstSectionBuildPromise = abortablePromiseWithExtractedComponents(splatSceneDownloadPromise.abortHandler); const splatSceneDownloadAndBuildPromise = abortablePromiseWithExtractedComponents(); this.addSplatSceneDownloadPromise(splatSceneDownloadPromise); From 093ca0b8a7700b3be1bfe29a9f0c93599e53b36f Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Thu, 29 Aug 2024 18:08:43 -0700 Subject: [PATCH 03/25] Add function to change spherical harmonics degree at runtime --- src/DropInViewer.js | 4 ++++ src/Viewer.js | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/src/DropInViewer.js b/src/DropInViewer.js index b56c2c94..764b560e 100644 --- a/src/DropInViewer.js +++ b/src/DropInViewer.js @@ -112,6 +112,10 @@ export class DropInViewer extends THREE.Group { return this.viewer.getSceneCount(); } + setActiveSphericalHarmonicsDegrees(activeSphericalHarmonicsDegrees) { + this.viewer.setActiveSphericalHarmonicsDegrees(activeSphericalHarmonicsDegrees); + } + dispose() { return this.viewer.dispose(); } diff --git a/src/Viewer.js b/src/Viewer.js index 2594af1f..9256c902 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -424,6 +424,11 @@ export class Viewer { this.renderMode = renderMode; } + setActiveSphericalHarmonicsDegrees(activeSphericalHarmonicsDegrees) { + this.splatMesh.material.uniforms.sphericalHarmonicsDegree.value = activeSphericalHarmonicsDegrees; + this.splatMesh.material.uniformsNeedUpdate = true; + } + onSplatMeshChanged(callback) { this.onSplatMeshChangedCallback = callback; } From d52d428d5a230f1fa78cf5608eb211cbe466b142 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Thu, 29 Aug 2024 20:40:44 -0700 Subject: [PATCH 04/25] Add parameter to control scene reveal speed --- README.md | 1 + src/Viewer.js | 5 ++++- src/splatmesh/SplatMesh.js | 8 +++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 59290a8a..4b7b6d37 100644 --- a/README.md +++ b/README.md @@ -330,6 +330,7 @@ Advanced `Viewer` parameters | `plyInMemoryCompressionLevel` | Level to compress `.ply` files when loading them for direct rendering (not exporting to `.ksplat`). Valid values are the same as `.ksplat` compression levels (0, 1, or 2). Default is 2. | `freeIntermediateSplatData` | When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and used to populate data textures will be freed. This will reduces memory usage, but if that data needs to be modified it will need to be re-populated from the splat buffer(s). Defaults to `false`. | `splatRenderMode` | Determine which splat rendering mode to enable. Valid values are defined in the `SplatRenderMode` enum: `ThreeD` and `TwoD`. `ThreeD` is the original/traditional mode and `TwoD` is the new mode described here: https://surfsplatting.github.io/ +| `sceneFadeInRateMultiplier` | Customize the speed at which the scene is revealed. Default is 1.0.
### Creating KSPLAT files diff --git a/src/Viewer.js b/src/Viewer.js index 9256c902..12ed0881 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -187,6 +187,9 @@ export class Viewer { } this.splatRenderMode = options.splatRenderMode; + // Customize the speed at which the scene is revealed + this.sceneFadeInRateMultiplier = options.sceneFadeInRateMultiplier || 1.0; + this.onSplatMeshChangedCallback = null; this.createSplatMesh(); @@ -265,7 +268,7 @@ export class Viewer { this.splatMesh = new SplatMesh(this.splatRenderMode, this.dynamicScene, this.enableOptionalEffects, this.halfPrecisionCovariancesOnGPU, this.devicePixelRatio, this.gpuAcceleratedSort, this.integerBasedSort, this.antialiased, this.maxScreenSpaceSplatSize, this.logLevel, - this.sphericalHarmonicsDegree); + this.sphericalHarmonicsDegree, this.sceneFadeInRateMultiplier); this.splatMesh.frustumCulled = false; if (this.onSplatMeshChangedCallback) this.onSplatMeshChangedCallback(); } diff --git a/src/splatmesh/SplatMesh.js b/src/splatmesh/SplatMesh.js index f44d3ac8..bea7ed2d 100644 --- a/src/splatmesh/SplatMesh.js +++ b/src/splatmesh/SplatMesh.js @@ -49,7 +49,7 @@ export class SplatMesh extends THREE.Mesh { constructor(splatRenderMode = SplatRenderMode.ThreeD, dynamicMode = false, enableOptionalEffects = false, halfPrecisionCovariancesOnGPU = false, devicePixelRatio = 1, enableDistancesComputationOnGPU = true, integerBasedDistancesComputation = false, antialiased = false, maxScreenSpaceSplatSize = 1024, logLevel = LogLevel.None, - sphericalHarmonicsDegree = 0) { + sphericalHarmonicsDegree = 0, sceneFadeInRateMultiplier = 1.0) { super(dummyGeometry, dummyMaterial); // Reference to a Three.js renderer @@ -98,6 +98,8 @@ export class SplatMesh extends THREE.Mesh { this.sphericalHarmonicsDegree = sphericalHarmonicsDegree; this.minSphericalHarmonicsDegree = 0; + this.sceneFadeInRateMultiplier = sceneFadeInRateMultiplier; + // The individual splat scenes stored in this splat mesh, each containing their own transform this.scenes = []; @@ -1193,8 +1195,8 @@ export class SplatMesh extends THREE.Mesh { } updateVisibleRegionFadeDistance(sceneRevealMode = SceneRevealMode.Default) { - const fastFadeRate = SCENE_FADEIN_RATE_FAST; - const gradualFadeRate = SCENE_FADEIN_RATE_GRADUAL; + const fastFadeRate = SCENE_FADEIN_RATE_FAST * this.sceneFadeInRateMultiplier; + const gradualFadeRate = SCENE_FADEIN_RATE_GRADUAL * this.sceneFadeInRateMultiplier; const defaultFadeInRate = this.finalBuild ? fastFadeRate : gradualFadeRate; const fadeInRate = sceneRevealMode === SceneRevealMode.Default ? defaultFadeInRate : gradualFadeRate; this.visibleRegionFadeStartRadius = (this.visibleRegionRadius - this.visibleRegionFadeStartRadius) * From 4be4448923ecd059d5d49c702bccb94a3071d067 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Fri, 30 Aug 2024 15:51:25 -0700 Subject: [PATCH 05/25] Added 'webXRSessionInit' parameter for passing WebXR initialization options to the viewer --- README.md | 1 + src/Viewer.js | 10 ++++++---- src/webxr/VRButton.js | 18 +++++++++++++----- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4b7b6d37..9e831929 100644 --- a/README.md +++ b/README.md @@ -320,6 +320,7 @@ Advanced `Viewer` parameters | `halfPrecisionCovariancesOnGPU` | Tells the viewer to use 16-bit floating point values when storing splat covariance data in textures, instead of 32-bit. Defaults to `false`. | `dynamicScene` | Tells the viewer to not make any optimizations that depend on the scene being static. Additionally all splat data retrieved from the viewer's splat mesh will not have their respective scene transform applied to them by default. | `webXRMode` | Tells the viewer whether or not to enable built-in Web VR or Web AR. Valid values are defined in the `WebXRMode` enum: `None`, `VR`, and `AR`. Defaults to `None`. +| `webXRSessionInit` | Tells the viewer to build a WebXR session with some options. Defaults with {}. For more details : https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/requestSession#options | `renderMode` | Controls when the viewer renders the scene. Valid values are defined in the `RenderMode` enum: `Always`, `OnChange`, and `Never`. Defaults to `Always`. | `sceneRevealMode` | Controls the fade-in effect used when the scene is loaded. Valid values are defined in the `SceneRevealMode` enum: `Default`, `Gradual`, and `Instant`. `Default` results in a nice, slow fade-in effect for progressively loaded scenes, and a fast fade-in for non progressively loaded scenes. `Gradual` will force a slow fade-in for all scenes. `Instant` will force all loaded scene data to be immediately visible. | `antialiased` | When true, will perform additional steps during rendering to address artifacts caused by the rendering of gaussians at substantially different resolutions than that at which they were rendered during training. This will only work correctly for models that were trained using a process that utilizes this compensation calculation. For more details: https://github.com/nerfstudio-project/gsplat/pull/117, https://github.com/graphdeco-inria/gaussian-splatting/issues/294#issuecomment-1772688093 diff --git a/src/Viewer.js b/src/Viewer.js index 12ed0881..c19abc63 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -122,6 +122,8 @@ export class Viewer { } this.webXRActive = false; + this.webXRSessionInit = options.webXRSessionInit || {}; + // if 'renderMode' is RenderMode.Always, then the viewer will rrender the scene on every update. If it is RenderMode.OnChange, // it will only render when something in the scene has changed. this.renderMode = options.renderMode || RenderMode.Always; @@ -291,7 +293,7 @@ export class Viewer { this.setupCamera(); this.setupRenderer(); - this.setupWebXR(); + this.setupWebXR(this.webXRSessionInit); this.setupControls(); this.setupEventHandlers(); @@ -348,12 +350,12 @@ export class Viewer { } - setupWebXR() { + setupWebXR(webXRSessionInit) { if (this.webXRMode) { if (this.webXRMode === WebXRMode.VR) { - this.rootElement.appendChild(VRButton.createButton(this.renderer)); + this.rootElement.appendChild(VRButton.createButton(this.renderer, webXRSessionInit)); } else if (this.webXRMode === WebXRMode.AR) { - this.rootElement.appendChild(ARButton.createButton(this.renderer)); + this.rootElement.appendChild(ARButton.createButton(this.renderer, webXRSessionInit)); } this.renderer.xr.addEventListener('sessionstart', (e) => { this.webXRActive = true; diff --git a/src/webxr/VRButton.js b/src/webxr/VRButton.js index 08208d75..5732eb92 100644 --- a/src/webxr/VRButton.js +++ b/src/webxr/VRButton.js @@ -14,7 +14,7 @@ all copies or substantial portions of the Software. export class VRButton { - static createButton( renderer ) { + static createButton( renderer, sessionInit = {} ) { const button = document.createElement( 'button' ); @@ -60,7 +60,15 @@ export class VRButton { // ('local' is always available for immersive sessions and doesn't need to // be requested separately.) - const sessionInit = { optionalFeatures: ['local-floor', 'bounded-floor', 'hand-tracking', 'layers'] }; + const sessionOptions = { + ...sessionInit, + optionalFeatures: [ + 'local-floor', + 'bounded-floor', + 'layers', + ...( sessionInit.optionalFeatures || [] ) + ], + }; button.onmouseenter = function() { @@ -78,7 +86,7 @@ export class VRButton { if ( currentSession === null ) { - navigator.xr.requestSession( 'immersive-vr', sessionInit ).then( onSessionStarted ); + navigator.xr.requestSession( 'immersive-vr', sessionOptions ).then( onSessionStarted ); } else { @@ -86,7 +94,7 @@ export class VRButton { if ( navigator.xr.offerSession !== undefined ) { - navigator.xr.offerSession( 'immersive-vr', sessionInit ) + navigator.xr.offerSession( 'immersive-vr', sessionOptions ) .then( onSessionStarted ) .catch( ( err ) => { @@ -102,7 +110,7 @@ export class VRButton { if ( navigator.xr.offerSession !== undefined ) { - navigator.xr.offerSession( 'immersive-vr', sessionInit ) + navigator.xr.offerSession( 'immersive-vr', sessionOptions ) .then( onSessionStarted ) .catch( ( err ) => { From aae2ecc642b1b818407dd00bc04abd8a3e727b5c Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sat, 31 Aug 2024 14:45:33 -0700 Subject: [PATCH 06/25] Reduce memory usage for PLY load --- src/loaders/ply/INRIAV1PlyParser.js | 10 + .../ply/PlayCanvasCompressedPlyParser.js | 16 ++ src/loaders/ply/PlyLoader.js | 203 ++++++++++-------- 3 files changed, 142 insertions(+), 87 deletions(-) diff --git a/src/loaders/ply/INRIAV1PlyParser.js b/src/loaders/ply/INRIAV1PlyParser.js index 4fbaed82..1351bdf6 100644 --- a/src/loaders/ply/INRIAV1PlyParser.js +++ b/src/loaders/ply/INRIAV1PlyParser.js @@ -82,6 +82,16 @@ export class INRIAV1PlyParser { } } + parseToUncompressedSplatArraySection(header, fromSplat, toSplat, splatData, splatDataOffset, + splatArray, outSphericalHarmonicsDegree = 0) { + outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); + for (let i = fromSplat; i <= toSplat; i++) { + const parsedSplat = INRIAV1PlyParser.parseToUncompressedSplat(splatData, i, header, + splatDataOffset, outSphericalHarmonicsDegree); + splatArray.addSplat(parsedSplat); + } + } + decodeSectionSplatData(sectionSplatData, splatCount, sectionHeader, outSphericalHarmonicsDegree) { outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, sectionHeader.sphericalHarmonicsDegree); const splatArray = new UncompressedSplatArray(outSphericalHarmonicsDegree); diff --git a/src/loaders/ply/PlayCanvasCompressedPlyParser.js b/src/loaders/ply/PlayCanvasCompressedPlyParser.js index 17cde87b..cd57eb92 100644 --- a/src/loaders/ply/PlayCanvasCompressedPlyParser.js +++ b/src/loaders/ply/PlayCanvasCompressedPlyParser.js @@ -378,6 +378,22 @@ export class PlayCanvasCompressedPlyParser { } } + static parseToUncompressedSplatArraySection(chunkElement, vertexElement, fromIndex, toIndex, chunkSplatIndexOffset, + vertexDataBuffer, veretxReadOffset, splatArray, propertyFilter = null) { + + PlayCanvasCompressedPlyParser.readElementData(vertexElement, vertexDataBuffer, veretxReadOffset, fromIndex, toIndex, propertyFilter); + + const { positionExtremes, scaleExtremes, position, rotation, scale, color } = + PlayCanvasCompressedPlyParser.getElementStorageArrays(chunkElement, vertexElement); + + for (let i = fromIndex; i <= toIndex; ++i) { + const tempSplat = UncompressedSplatArray.createSplat(); + PlayCanvasCompressedPlyParser.decompressSplat(i, chunkSplatIndexOffset, position, positionExtremes, + scale, scaleExtremes, rotation, color, tempSplat); + splatArray.addSplat(tempSplat); + } + } + static parseToUncompressedSplatArray(plyBuffer) { const { chunkElement, vertexElement } = PlayCanvasCompressedPlyParser.readPly(plyBuffer); diff --git a/src/loaders/ply/PlyLoader.js b/src/loaders/ply/PlyLoader.js index 8ad46c54..acf4713a 100644 --- a/src/loaders/ply/PlyLoader.js +++ b/src/loaders/ply/PlyLoader.js @@ -9,6 +9,7 @@ import { SplatBuffer } from '../SplatBuffer.js'; import { SplatBufferGenerator } from '../SplatBufferGenerator.js'; import { LoaderStatus } from '../LoaderStatus.js'; import { Constants } from '../../Constants.js'; +import { UncompressedSplatArray } from '../UncompressedSplatArray.js'; function storeChunksInBuffer(chunks, buffer) { let inBytes = 0; @@ -29,7 +30,7 @@ function storeChunksInBuffer(chunks, buffer) { export class PlyLoader { - static loadFromURL(fileName, onProgress, progressiveLoad, onStreamedSectionProgress, minimumAlpha, compressionLevel, + static loadFromURL(fileName, onProgress, progressiveLoad, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, outSphericalHarmonicsDegree = 0, sectionSize, sceneCenter, blockSize, bucketSize) { const progressiveLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; @@ -47,7 +48,7 @@ export class PlyLoader { let readyToLoadSplatData = false; let compressed = false; - const progressiveLoadPromise = nativePromiseWithExtractedComponents(); + const loadPromise = nativePromiseWithExtractedComponents(); let numBytesStreamed = 0; let numBytesParsed = 0; @@ -56,44 +57,46 @@ export class PlyLoader { let header = null; let chunks = []; - const textDecoder = new TextDecoder(); + const standardLoadUncompressedSplatArray = new UncompressedSplatArray(); + const textDecoder = new TextDecoder(); const inriaV1PlyParser = new INRIAV1PlyParser(); const localOnProgress = (percent, percentLabel, chunkData) => { const loadComplete = percent >= 100; - if (progressiveLoad) { - - if (chunkData) { - chunks.push({ - 'data': chunkData, - 'sizeBytes': chunkData.byteLength, - 'startBytes': numBytesDownloaded, - 'endBytes': numBytesDownloaded + chunkData.byteLength - }); - numBytesDownloaded += chunkData.byteLength; - } - if (!headerLoaded) { - headerText += textDecoder.decode(chunkData); - if (PlyParserUtils.checkTextForEndHeader(headerText)) { - const plyFormat = PlyParserUtils.determineHeaderFormatFromHeaderText(headerText); - if (plyFormat === PlyFormat.INRIAV1) { - header = inriaV1PlyParser.decodeHeaderText(headerText); - maxSplatCount = header.splatCount; - readyToLoadSplatData = true; - compressed = false; - } else if (plyFormat === PlyFormat.PlayCanvasCompressed) { - header = PlayCanvasCompressedPlyParser.decodeHeaderText(headerText); - maxSplatCount = header.vertexElement.count; - compressed = true; - } else { - throw new Error('PlyLoader.loadFromURL() -> Selected Ply format cannot be progressively loaded.'); - } - outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); + if (chunkData) { + chunks.push({ + 'data': chunkData, + 'sizeBytes': chunkData.byteLength, + 'startBytes': numBytesDownloaded, + 'endBytes': numBytesDownloaded + chunkData.byteLength + }); + numBytesDownloaded += chunkData.byteLength; + } - const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; - const splatBufferSizeBytes = splatDataOffsetBytes + shDescriptor.BytesPerSplat * maxSplatCount; + if (!headerLoaded) { + headerText += textDecoder.decode(chunkData); + if (PlyParserUtils.checkTextForEndHeader(headerText)) { + const plyFormat = PlyParserUtils.determineHeaderFormatFromHeaderText(headerText); + if (plyFormat === PlyFormat.INRIAV1) { + header = inriaV1PlyParser.decodeHeaderText(headerText); + maxSplatCount = header.splatCount; + readyToLoadSplatData = true; + compressed = false; + } else if (plyFormat === PlyFormat.PlayCanvasCompressed) { + header = PlayCanvasCompressedPlyParser.decodeHeaderText(headerText); + maxSplatCount = header.vertexElement.count; + compressed = true; + } else { + throw new Error('PlyLoader.loadFromURL() -> Selected Ply format cannot be progressively loaded.'); + } + outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); + + const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; + const splatBufferSizeBytes = splatDataOffsetBytes + shDescriptor.BytesPerSplat * maxSplatCount; + + if (progressiveLoad) { progressiveLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); SplatBuffer.writeHeaderToBuffer({ versionMajor: SplatBuffer.CurrentMajorVersion, @@ -105,42 +108,44 @@ export class PlyLoader { compressionLevel: 0, sceneCenter: new THREE.Vector3() }, progressiveLoadBufferOut); - - numBytesStreamed = header.headerSizeBytes; - numBytesParsed = header.headerSizeBytes; - headerLoaded = true; - } - } else if (compressed && !readyToLoadSplatData) { - const sizeRequiredForHeaderAndChunks = header.headerSizeBytes + header.chunkElement.storageSizeBytes; - compressedPlyHeaderChunksBuffer = storeChunksInBuffer(chunks, compressedPlyHeaderChunksBuffer); - if (compressedPlyHeaderChunksBuffer.byteLength >= sizeRequiredForHeaderAndChunks) { - PlayCanvasCompressedPlyParser.readElementData(header.chunkElement, compressedPlyHeaderChunksBuffer, - header.headerSizeBytes); - numBytesStreamed = sizeRequiredForHeaderAndChunks; - numBytesParsed = sizeRequiredForHeaderAndChunks; - readyToLoadSplatData = true; } + + numBytesStreamed = header.headerSizeBytes; + numBytesParsed = header.headerSizeBytes; + headerLoaded = true; } + } else if (compressed && !readyToLoadSplatData) { + const sizeRequiredForHeaderAndChunks = header.headerSizeBytes + header.chunkElement.storageSizeBytes; + compressedPlyHeaderChunksBuffer = storeChunksInBuffer(chunks, compressedPlyHeaderChunksBuffer); + if (compressedPlyHeaderChunksBuffer.byteLength >= sizeRequiredForHeaderAndChunks) { + PlayCanvasCompressedPlyParser.readElementData(header.chunkElement, compressedPlyHeaderChunksBuffer, + header.headerSizeBytes); + numBytesStreamed = sizeRequiredForHeaderAndChunks; + numBytesParsed = sizeRequiredForHeaderAndChunks; + readyToLoadSplatData = true; + } + } - if (headerLoaded && readyToLoadSplatData) { + if (headerLoaded && readyToLoadSplatData) { - if (chunks.length > 0) { + if (chunks.length > 0) { - progressiveLoadBufferIn = storeChunksInBuffer(chunks, progressiveLoadBufferIn); + progressiveLoadBufferIn = storeChunksInBuffer(chunks, progressiveLoadBufferIn); - const bytesLoadedSinceLastStreamedSection = numBytesDownloaded - numBytesStreamed; - if (bytesLoadedSinceLastStreamedSection > progressiveLoadSectionSizeBytes || loadComplete) { - const numBytesToProcess = numBytesDownloaded - numBytesParsed; - const addedSplatCount = Math.floor(numBytesToProcess / header.bytesPerSplat); - const numBytesToParse = addedSplatCount * header.bytesPerSplat; - const numBytesLeftOver = numBytesToProcess - numBytesToParse; - const newSplatCount = splatCount + addedSplatCount; - const parsedDataViewOffset = numBytesParsed - chunks[0].startBytes; - const dataToParse = new DataView(progressiveLoadBufferIn, parsedDataViewOffset, numBytesToParse); + const bytesLoadedSinceLastStreamedSection = numBytesDownloaded - numBytesStreamed; + if (bytesLoadedSinceLastStreamedSection > progressiveLoadSectionSizeBytes || loadComplete) { + const numBytesToProcess = numBytesDownloaded - numBytesParsed; + const addedSplatCount = Math.floor(numBytesToProcess / header.bytesPerSplat); + const numBytesToParse = addedSplatCount * header.bytesPerSplat; + const numBytesLeftOver = numBytesToProcess - numBytesToParse; + const newSplatCount = splatCount + addedSplatCount; + const parsedDataViewOffset = numBytesParsed - chunks[0].startBytes; + const dataToParse = new DataView(progressiveLoadBufferIn, parsedDataViewOffset, numBytesToParse); - const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; - const outOffset = splatCount * shDescriptor.BytesPerSplat + splatDataOffsetBytes; + const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; + const outOffset = splatCount * shDescriptor.BytesPerSplat + splatDataOffsetBytes; + if (progressiveLoad) { if (compressed) { PlayCanvasCompressedPlyParser.parseToUncompressedSplatBufferSection(header.chunkElement, header.vertexElement, 0, @@ -152,8 +157,23 @@ export class PlyLoader { 0, progressiveLoadBufferOut, outOffset, outSphericalHarmonicsDegree); } + } else { + if (compressed) { + PlayCanvasCompressedPlyParser.parseToUncompressedSplatArraySection(header.chunkElement, + header.vertexElement, 0, + addedSplatCount - 1, splatCount, + dataToParse, 0, + standardLoadUncompressedSplatArray); + } else { + inriaV1PlyParser.parseToUncompressedSplatArraySection(header, 0, addedSplatCount - 1, dataToParse, + 0, standardLoadUncompressedSplatArray, + outSphericalHarmonicsDegree); + } + } - splatCount = newSplatCount; + splatCount = newSplatCount; + + if (progressiveLoad) { if (!progressiveLoadSplatBuffer) { SplatBuffer.writeSectionHeaderToBuffer({ maxSplatCount: maxSplatCount, @@ -170,44 +190,53 @@ export class PlyLoader { progressiveLoadSplatBuffer = new SplatBuffer(progressiveLoadBufferOut, false); } progressiveLoadSplatBuffer.updateLoadedCounts(1, splatCount); - onStreamedSectionProgress(progressiveLoadSplatBuffer, loadComplete); - numBytesStreamed += progressiveLoadSectionSizeBytes; - numBytesParsed += numBytesToParse; + onProgressiveLoadSectionProgress(progressiveLoadSplatBuffer, loadComplete); + } - if (numBytesLeftOver === 0) { - chunks = []; - } else { - let keepChunks = []; - let keepSize = 0; - for (let i = chunks.length - 1; i >= 0; i--) { - const chunk = chunks[i]; - keepSize += chunk.sizeBytes; - keepChunks.unshift(chunk); - if (keepSize >= numBytesLeftOver) break; - } - chunks = keepChunks; + numBytesStreamed += progressiveLoadSectionSizeBytes; + numBytesParsed += numBytesToParse; + + if (numBytesLeftOver === 0) { + chunks = []; + } else { + let keepChunks = []; + let keepSize = 0; + for (let i = chunks.length - 1; i >= 0; i--) { + const chunk = chunks[i]; + keepSize += chunk.sizeBytes; + keepChunks.unshift(chunk); + if (keepSize >= numBytesLeftOver) break; } + chunks = keepChunks; } } + } - if (loadComplete) { - progressiveLoadPromise.resolve(progressiveLoadSplatBuffer); + if (loadComplete) { + if (progressiveLoad) { + loadPromise.resolve(progressiveLoadSplatBuffer); + } else { + loadPromise.resolve(standardLoadUncompressedSplatArray); } } - } + if (onProgress) onProgress(percent, percentLabel, LoaderStatus.Downloading); }; - return fetchWithProgress(fileName, localOnProgress, !progressiveLoad).then((plyFileData) => { + return fetchWithProgress(fileName, localOnProgress, false).then(() => { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); - const loadPromise = progressiveLoad ? progressiveLoadPromise.promise : - PlyLoader.loadFromFileData(plyFileData, minimumAlpha, compressionLevel, outSphericalHarmonicsDegree, - sectionSize, sceneCenter, blockSize, bucketSize); - return loadPromise.then((splatBuffer) => { - if (onProgress) onProgress(100, '100%', LoaderStatus.Done); - return splatBuffer; + return loadPromise.promise.then((splatData) => { + if (progressiveLoad) { + if (onProgress) onProgress(100, '100%', LoaderStatus.Done); + return splatData; + } else { + const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, sectionSize, + sceneCenter, blockSize, bucketSize); + return splatBufferGenerator.generateFromUncompressedSplatArray(splatData); + } }); + }); } From 1ccb5208820557ad71fc1edd5e143aa26ee70d7b Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sat, 31 Aug 2024 18:29:51 -0700 Subject: [PATCH 07/25] Reduce memory usage for SPLAT load --- src/loaders/ply/PlyLoader.js | 16 +++--- src/loaders/splat/SplatLoader.js | 93 ++++++++++++++++++++++---------- src/loaders/splat/SplatParser.js | 19 +++++++ 3 files changed, 93 insertions(+), 35 deletions(-) diff --git a/src/loaders/ply/PlyLoader.js b/src/loaders/ply/PlyLoader.js index acf4713a..7cfa51a7 100644 --- a/src/loaders/ply/PlyLoader.js +++ b/src/loaders/ply/PlyLoader.js @@ -57,7 +57,7 @@ export class PlyLoader { let header = null; let chunks = []; - const standardLoadUncompressedSplatArray = new UncompressedSplatArray(); + let standardLoadUncompressedSplatArray; const textDecoder = new TextDecoder(); const inriaV1PlyParser = new INRIAV1PlyParser(); @@ -108,6 +108,8 @@ export class PlyLoader { compressionLevel: 0, sceneCenter: new THREE.Vector3() }, progressiveLoadBufferOut); + } else { + standardLoadUncompressedSplatArray = new UncompressedSplatArray(outSphericalHarmonicsDegree); } numBytesStreamed = header.headerSizeBytes; @@ -224,19 +226,21 @@ export class PlyLoader { if (onProgress) onProgress(percent, percentLabel, LoaderStatus.Downloading); }; + if (onProgress) onProgress(0, '0%', LoaderStatus.Downloading); return fetchWithProgress(fileName, localOnProgress, false).then(() => { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); return loadPromise.promise.then((splatData) => { + if (onProgress) onProgress(100, '100%', LoaderStatus.Done); if (progressiveLoad) { - if (onProgress) onProgress(100, '100%', LoaderStatus.Done); return splatData; } else { - const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, sectionSize, - sceneCenter, blockSize, bucketSize); - return splatBufferGenerator.generateFromUncompressedSplatArray(splatData); + return delayedExecute(() => { + const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, sectionSize, + sceneCenter, blockSize, bucketSize); + return splatBufferGenerator.generateFromUncompressedSplatArray(splatData); + }); } }); - }); } diff --git a/src/loaders/splat/SplatLoader.js b/src/loaders/splat/SplatLoader.js index 5dd067c9..8bfb1330 100644 --- a/src/loaders/splat/SplatLoader.js +++ b/src/loaders/splat/SplatLoader.js @@ -3,12 +3,13 @@ import { SplatBuffer } from '../SplatBuffer.js'; import { SplatBufferGenerator } from '../SplatBufferGenerator.js'; import { SplatParser } from './SplatParser.js'; import { fetchWithProgress, delayedExecute, nativePromiseWithExtractedComponents } from '../../Util.js'; +import { UncompressedSplatArray } from '../UncompressedSplatArray.js'; import { LoaderStatus } from '../LoaderStatus.js'; import { Constants } from '../../Constants.js'; export class SplatLoader { - static loadFromURL(fileName, onProgress, progressiveLoad, onStreamedSectionProgress, minimumAlpha, compressionLevel, + static loadFromURL(fileName, onProgress, progressiveLoad, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, optimizeSplatData, sectionSize, sceneCenter, blockSize, bucketSize) { const splatDataOffsetBytes = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes; @@ -21,7 +22,9 @@ export class SplatLoader { let maxSplatCount = 0; let splatCount = 0; - const progressiveLoadPromise = nativePromiseWithExtractedComponents(); + let standardLoadUncompressedSplatArray; + + const loadPromise = nativePromiseWithExtractedComponents(); let numBytesStreamed = 0; let numBytesLoaded = 0; @@ -30,12 +33,14 @@ export class SplatLoader { const localOnProgress = (percent, percentStr, chunk, fileSize) => { const loadComplete = percent >= 100; if (!fileSize) progressiveLoad = false; - if (progressiveLoad) { - if (!progressiveLoadBufferIn) { - maxSplatCount = fileSize / SplatParser.RowSizeBytes; - progressiveLoadBufferIn = new ArrayBuffer(fileSize); - const bytesPerSplat = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[0].BytesPerSplat; - const splatBufferSizeBytes = splatDataOffsetBytes + bytesPerSplat * maxSplatCount; + + if (!progressiveLoadBufferIn) { + maxSplatCount = fileSize / SplatParser.RowSizeBytes; + progressiveLoadBufferIn = new ArrayBuffer(fileSize); + const bytesPerSplat = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[0].BytesPerSplat; + const splatBufferSizeBytes = splatDataOffsetBytes + bytesPerSplat * maxSplatCount; + + if (progressiveLoad) { progressiveLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); SplatBuffer.writeHeaderToBuffer({ versionMajor: SplatBuffer.CurrentMajorVersion, @@ -47,21 +52,33 @@ export class SplatLoader { compressionLevel: 0, sceneCenter: new THREE.Vector3() }, progressiveLoadBufferOut); + } else { + standardLoadUncompressedSplatArray = new UncompressedSplatArray(0); } + } - if (chunk) { - chunks.push(chunk); - new Uint8Array(progressiveLoadBufferIn, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); - numBytesLoaded += chunk.byteLength; + if (chunk) { + chunks.push(chunk); + new Uint8Array(progressiveLoadBufferIn, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); + numBytesLoaded += chunk.byteLength; - const bytesLoadedSinceLastSection = numBytesLoaded - numBytesStreamed; - if (bytesLoadedSinceLastSection > progressiveLoadSectionSizeBytes || loadComplete) { - const bytesToUpdate = loadComplete ? bytesLoadedSinceLastSection : progressiveLoadSectionSizeBytes; - const addedSplatCount = bytesToUpdate / SplatParser.RowSizeBytes; - const newSplatCount = splatCount + addedSplatCount; + const bytesLoadedSinceLastSection = numBytesLoaded - numBytesStreamed; + if (bytesLoadedSinceLastSection > progressiveLoadSectionSizeBytes || loadComplete) { + const bytesToUpdate = loadComplete ? bytesLoadedSinceLastSection : progressiveLoadSectionSizeBytes; + const addedSplatCount = bytesToUpdate / SplatParser.RowSizeBytes; + const newSplatCount = splatCount + addedSplatCount; + + if (progressiveLoad) { SplatParser.parseToUncompressedSplatBufferSection(splatCount, newSplatCount - 1, progressiveLoadBufferIn, 0, - progressiveLoadBufferOut, splatDataOffsetBytes); - splatCount = newSplatCount; + progressiveLoadBufferOut, splatDataOffsetBytes); + } else { + SplatParser.parseToUncompressedSplatArraySection(splatCount, newSplatCount - 1, progressiveLoadBufferIn, 0, + standardLoadUncompressedSplatArray); + } + + splatCount = newSplatCount; + + if (progressiveLoad) { if (!progressiveLoadSplatBuffer) { SplatBuffer.writeSectionHeaderToBuffer({ maxSplatCount: maxSplatCount, @@ -77,26 +94,44 @@ export class SplatLoader { progressiveLoadSplatBuffer = new SplatBuffer(progressiveLoadBufferOut, false); } progressiveLoadSplatBuffer.updateLoadedCounts(1, splatCount); - onStreamedSectionProgress(progressiveLoadSplatBuffer, loadComplete); - numBytesStreamed += progressiveLoadSectionSizeBytes; + onProgressiveLoadSectionProgress(progressiveLoadSplatBuffer, loadComplete); } + + numBytesStreamed += progressiveLoadSectionSizeBytes; } - if (loadComplete) { - progressiveLoadPromise.resolve(progressiveLoadSplatBuffer); + } + + if (loadComplete) { + if (progressiveLoad) { + loadPromise.resolve(progressiveLoadSplatBuffer); + } else { + loadPromise.resolve(standardLoadUncompressedSplatArray); } } + if (onProgress) onProgress(percent, percentStr, LoaderStatus.Downloading); return progressiveLoad; }; - return fetchWithProgress(fileName, localOnProgress, true).then((fullBuffer) => { + if (onProgress) onProgress(0, '0%', LoaderStatus.Downloading); + return fetchWithProgress(fileName, localOnProgress, true).then(() => { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); - const loadPromise = progressiveLoad ? progressiveLoadPromise.promise : - SplatLoader.loadFromFileData(fullBuffer, minimumAlpha, compressionLevel, optimizeSplatData, - sectionSize, sceneCenter, blockSize, bucketSize); - return loadPromise.then((splatBuffer) => { + return loadPromise.promise.then((splatData) => { if (onProgress) onProgress(100, '100%', LoaderStatus.Done); - return splatBuffer; + if (progressiveLoad) { + return splatData; + } else { + return delayedExecute(() => { + if (optimizeSplatData) { + const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, + sectionSize, sceneCenter, blockSize, + bucketSize); + return splatBufferGenerator.generateFromUncompressedSplatArray(splatData); + } else { + return SplatBuffer.generateFromUncompressedSplatArrays([splatData], minimumAlpha, 0, new THREE.Vector3()); + } + }); + } }); }); } diff --git a/src/loaders/splat/SplatParser.js b/src/loaders/splat/SplatParser.js index 444779ae..53c1c152 100644 --- a/src/loaders/splat/SplatParser.js +++ b/src/loaders/splat/SplatParser.js @@ -55,6 +55,25 @@ export class SplatParser { } } + static parseToUncompressedSplatArraySection(fromSplat, toSplat, fromBuffer, fromOffset, splatArray) { + + for (let i = fromSplat; i <= toSplat; i++) { + const inBase = i * SplatParser.RowSizeBytes + fromOffset; + const inCenter = new Float32Array(fromBuffer, inBase, 3); + const inScale = new Float32Array(fromBuffer, inBase + SplatParser.CenterSizeBytes, 3); + const inColor = new Uint8Array(fromBuffer, inBase + SplatParser.CenterSizeBytes + SplatParser.ScaleSizeBytes, 4); + const inRotation = new Uint8Array(fromBuffer, inBase + SplatParser.CenterSizeBytes + SplatParser.ScaleSizeBytes + + SplatParser.RotationSizeBytes, 4); + + const quat = new THREE.Quaternion((inRotation[1] - 128) / 128, (inRotation[2] - 128) / 128, + (inRotation[3] - 128) / 128, (inRotation[0] - 128) / 128); + quat.normalize(); + + splatArray.addSplatFromComonents(inCenter[0], inCenter[1], inCenter[2], inScale[0], inScale[1], inScale[2], + quat.w, quat.x, quat.y, quat.z, inColor[0], inColor[1], inColor[2], inColor[3]); + } + } + static parseStandardSplatToUncompressedSplatArray(inBuffer) { // Standard .splat row layout: // XYZ - Position (Float32) From b0eeceeb10300bbacc9fb78ce6ecc521e04465e2 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sat, 31 Aug 2024 18:54:43 -0700 Subject: [PATCH 08/25] Added 'splatInMemoryCompressionLevel' parameter --- README.md | 13 ++++++--- src/Util.js | 8 +++--- src/Viewer.js | 44 +++++++++++++++++++++--------- src/loaders/DirectLoadError.js | 7 +++++ src/loaders/ply/PlyLoader.js | 32 +++++++++++++++------- src/loaders/splat/SplatLoader.js | 46 ++++++++++++++++++-------------- 6 files changed, 101 insertions(+), 49 deletions(-) create mode 100644 src/loaders/DirectLoadError.js diff --git a/README.md b/README.md index 9e831929..d7af3ee7 100644 --- a/README.md +++ b/README.md @@ -288,7 +288,7 @@ const viewer = new GaussianSplats3D.Viewer({ 'logLevel': GaussianSplats3D.LogLevel.None, 'sphericalHarmonicsDegree': 0, `enableOptionalEffects`: false, - `plyInMemoryCompressionLevel`: 2 + `inMemoryCompressionLevel`: 2 `freeIntermediateSplatData`: false }); viewer.addSplatScene('') @@ -328,7 +328,8 @@ Advanced `Viewer` parameters | `logLevel` | Verbosity of the console logging. Defaults to `GaussianSplats3D.LogLevel.None`. | `sphericalHarmonicsDegree` | Degree of spherical harmonics to utilize in rendering splats (assuming the data is present in the splat scene). Valid values are 0, 1, or 2. Default value is 0. | `enableOptionalEffects` | When true, allows for usage of extra properties and attributes during rendering for effects such as opacity adjustment. Default is `false` for performance reasons. These properties are separate from transform properties (scale, rotation, position) that are enabled by the `dynamicScene` parameter. -| `plyInMemoryCompressionLevel` | Level to compress `.ply` files when loading them for direct rendering (not exporting to `.ksplat`). Valid values are the same as `.ksplat` compression levels (0, 1, or 2). Default is 2. +| `inMemoryCompressionLevel` | Level to compress `.ply` or `.ksplat` files when loading them for direct rendering (not exporting to `.ksplat`). Valid values are the same as `.ksplat` compression levels (0, 1, or 2). Default is 0. +| `optimizeSplatData` | Reorder splat data in memory after loading is complete to optimize cache utilization. Default is `true`. Does not apply if splat scene is progressively loaded. | `freeIntermediateSplatData` | When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and used to populate data textures will be freed. This will reduces memory usage, but if that data needs to be modified it will need to be re-populated from the splat buffer(s). Defaults to `false`. | `splatRenderMode` | Determine which splat rendering mode to enable. Valid values are defined in the `SplatRenderMode` enum: `ThreeD` and `TwoD`. `ThreeD` is the original/traditional mode and `TwoD` is the new mode described here: https://surfsplatting.github.io/ | `sceneFadeInRateMultiplier` | Customize the speed at which the scene is revealed. Default is 1.0. @@ -344,9 +345,13 @@ const compressionLevel = 1; const splatAlphaRemovalThreshold = 5; // out of 255 const sphericalHarmonicsDegree = 1; GaussianSplats3D.PlyLoader.loadFromURL('', + onProgress, + progressiveLoad, + onProgressiveLoadSectionProgress, + minimumAlpha, compressionLevel, - splatAlphaRemovalThreshold, - sphericalHarmonicsDegree) + optimizeSplatData, + outSphericalHarmonicsDegree) .then((splatBuffer) => { GaussianSplats3D.KSplatLoader.downloadFile(splatBuffer, 'converted_file.ksplat'); }); diff --git a/src/Util.js b/src/Util.js index 5ab58f3d..24ac8375 100644 --- a/src/Util.js +++ b/src/Util.js @@ -103,10 +103,12 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { percent = bytesDownloaded / fileSize * 100; percentLabel = `${percent.toFixed(2)}%`; } - if (saveChunks) chunks.push(chunk); + if (saveChunks) { + chunks.push(chunk); + } if (onProgress) { - const cancelSaveChucnks = onProgress(percent, percentLabel, chunk, fileSize); - if (cancelSaveChucnks) saveChunks = false; + const cancelSaveChunks = onProgress(percent, percentLabel, chunk, fileSize); + if (cancelSaveChunks) saveChunks = false; } } catch (error) { reject(error); diff --git a/src/Viewer.js b/src/Viewer.js index c19abc63..3f217bf0 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -20,6 +20,7 @@ import { VRButton } from './webxr/VRButton.js'; import { ARButton } from './webxr/ARButton.js'; import { delayedExecute, abortablePromiseWithExtractedComponents } from './Util.js'; import { LoaderStatus } from './loaders/LoaderStatus.js'; +import { DirectLoadError } from '../DirectLoadError.js'; import { RenderMode } from './RenderMode.js'; import { LogLevel } from './LogLevel.js'; import { SceneRevealMode } from './SceneRevealMode.js'; @@ -158,10 +159,17 @@ export class Viewer { this.enableSIMDInSort = options.enableSIMDInSort; // Level to compress PLY files when loading them for direct rendering (not exporting to .ksplat) - if (options.plyInMemoryCompressionLevel === undefined || options.plyInMemoryCompressionLevel === null) { - options.plyInMemoryCompressionLevel = 2; + if (options.inMemoryCompressionLevel === undefined || options.inMemoryCompressionLevel === null) { + options.inMemoryCompressionLevel = 0; } - this.plyInMemoryCompressionLevel = options.plyInMemoryCompressionLevel; + this.inMemoryCompressionLevel = options.inMemoryCompressionLevel; + + // Reorder splat data in memory after loading is complete to optimize cache utilization. Default is true. + // Does not apply if splat scene is progressively loaded. + if (options.optimizeSplatData === undefined || options.optimizeSplatData === null) { + options.optimizeSplatData = true; + } + this.optimizeSplatData = options.optimizeSplatData; // When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and is used to // populate the data textures will be freed. This will reduces memory usage, but if that data needs to be modified @@ -728,7 +736,7 @@ export class Viewer { } const format = (options.format !== undefined && options.format !== null) ? options.format : sceneFormatFromPath(path); - const progressiveLoad = Viewer.isProgressivelyLoadable(format) && options.progressiveLoad; + const progressiveLoad = Viewer.isProgressivelyLoadable(format) && options.progressiveLoad && !this.optimizeSplatData; const showLoadingUI = (options.showLoadingUI !== undefined && options.showLoadingUI !== null) ? options.showLoadingUI : true; let loadingUITaskId = null; @@ -1032,14 +1040,24 @@ export class Viewer { */ downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold = 1, onProgress = undefined, progressiveBuild = false, onSectionBuilt = undefined, format) { - if (format === SceneFormat.Splat) { - return SplatLoader.loadFromURL(path, onProgress, progressiveBuild, - onSectionBuilt, splatAlphaRemovalThreshold, 0, false); - } else if (format === SceneFormat.KSplat) { - return KSplatLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt); - } else if (format === SceneFormat.Ply) { - return PlyLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, - splatAlphaRemovalThreshold, this.plyInMemoryCompressionLevel, this.sphericalHarmonicsDegree); + try { + if (format === SceneFormat.Splat) { + return SplatLoader.loadFromURL(path, onProgress, progressiveBuild, + onSectionBuilt, splatAlphaRemovalThreshold, + this.inMemoryCompressionLevel, this.optimizeSplatData); + } else if (format === SceneFormat.KSplat) { + return KSplatLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt); + } else if (format === SceneFormat.Ply) { + return PlyLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, + splatAlphaRemovalThreshold, this.inMemoryCompressionLevel, + this.optimizeSplatData, this.sphericalHarmonicsDegree); + } + } catch (e) { + if (e instanceof DirectLoadError) { + throw new Error('File type or server does not support progressive loading.'); + } else { + throw e; + } } throw new Error(`Viewer::downloadSplatSceneToSplatBuffer -> File format not supported: ${path}`); @@ -1177,7 +1195,7 @@ export class Viewer { if (showLoadingUIForSplatTreeBuild && splatCount >= MIN_SPLAT_COUNT_TO_SHOW_SPLAT_TREE_LOADING_SPINNER) { if (!finished && !splatOptimizingTaskId) { this.loadingSpinner.setMinimized(true, true); - splatOptimizingTaskId = this.loadingSpinner.addTask('Optimizing splats...'); + splatOptimizingTaskId = this.loadingSpinner.addTask('Optimizing data structures...'); } } }; diff --git a/src/loaders/DirectLoadError.js b/src/loaders/DirectLoadError.js new file mode 100644 index 00000000..1672d595 --- /dev/null +++ b/src/loaders/DirectLoadError.js @@ -0,0 +1,7 @@ +export class DirectLoadError extends Error { + + constructor(msg) { + super(msg); + } + +} diff --git a/src/loaders/ply/PlyLoader.js b/src/loaders/ply/PlyLoader.js index 7cfa51a7..ebb80635 100644 --- a/src/loaders/ply/PlyLoader.js +++ b/src/loaders/ply/PlyLoader.js @@ -8,6 +8,7 @@ import { fetchWithProgress, delayedExecute, nativePromiseWithExtractedComponents import { SplatBuffer } from '../SplatBuffer.js'; import { SplatBufferGenerator } from '../SplatBufferGenerator.js'; import { LoaderStatus } from '../LoaderStatus.js'; +import { DirectLoadError } from '../DirectLoadError.js'; import { Constants } from '../../Constants.js'; import { UncompressedSplatArray } from '../UncompressedSplatArray.js'; @@ -28,10 +29,23 @@ function storeChunksInBuffer(chunks, buffer) { return buffer; } +function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, sectionSize, sceneCenter, blockSize, bucketSize) { + if (optimizeSplatData) { + const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, + sectionSize, sceneCenter, + blockSize, bucketSize); + return splatBufferGenerator.generateFromUncompressedSplatArray(splatData); + } else { + return SplatBuffer.generateFromUncompressedSplatArrays([splatData], minimumAlpha, 0, new THREE.Vector3()); + } +} + export class PlyLoader { static loadFromURL(fileName, onProgress, progressiveLoad, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, - outSphericalHarmonicsDegree = 0, sectionSize, sceneCenter, blockSize, bucketSize) { + optimizeSplatData = true, outSphericalHarmonicsDegree = 0, sectionSize, sceneCenter, blockSize, bucketSize) { + + if (progressiveLoad) optimizeSplatData = false; const progressiveLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; const splatDataOffsetBytes = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes; @@ -89,7 +103,7 @@ export class PlyLoader { maxSplatCount = header.vertexElement.count; compressed = true; } else { - throw new Error('PlyLoader.loadFromURL() -> Selected Ply format cannot be progressively loaded.'); + throw new DirectLoadError('PlyLoader.loadFromURL() -> Selected Ply format cannot be directly loaded.'); } outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); @@ -192,7 +206,9 @@ export class PlyLoader { progressiveLoadSplatBuffer = new SplatBuffer(progressiveLoadBufferOut, false); } progressiveLoadSplatBuffer.updateLoadedCounts(1, splatCount); - onProgressiveLoadSectionProgress(progressiveLoadSplatBuffer, loadComplete); + if (onProgressiveLoadSectionProgress) { + onProgressiveLoadSectionProgress(progressiveLoadSplatBuffer, loadComplete); + } } numBytesStreamed += progressiveLoadSectionSizeBytes; @@ -235,9 +251,8 @@ export class PlyLoader { return splatData; } else { return delayedExecute(() => { - const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, sectionSize, - sceneCenter, blockSize, bucketSize); - return splatBufferGenerator.generateFromUncompressedSplatArray(splatData); + return finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, + sectionSize, sceneCenter, blockSize, bucketSize); }); } }); @@ -250,9 +265,8 @@ export class PlyLoader { return PlyParser.parseToUncompressedSplatArray(plyFileData, outSphericalHarmonicsDegree); }) .then((splatArray) => { - const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, sectionSize, - sceneCenter, blockSize, bucketSize); - return splatBufferGenerator.generateFromUncompressedSplatArray(splatArray); + return finalize(splatArray, optimizeSplatData, minimumAlpha, compressionLevel, + sectionSize, sceneCenter, blockSize, bucketSize); }); } } diff --git a/src/loaders/splat/SplatLoader.js b/src/loaders/splat/SplatLoader.js index 8bfb1330..70305ad2 100644 --- a/src/loaders/splat/SplatLoader.js +++ b/src/loaders/splat/SplatLoader.js @@ -5,12 +5,26 @@ import { SplatParser } from './SplatParser.js'; import { fetchWithProgress, delayedExecute, nativePromiseWithExtractedComponents } from '../../Util.js'; import { UncompressedSplatArray } from '../UncompressedSplatArray.js'; import { LoaderStatus } from '../LoaderStatus.js'; +import { DirectLoadError } from '../DirectLoadError.js'; import { Constants } from '../../Constants.js'; +function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, sectionSize, sceneCenter, blockSize, bucketSize) { + if (optimizeSplatData) { + const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, + sectionSize, sceneCenter, + blockSize, bucketSize); + return splatBufferGenerator.generateFromUncompressedSplatArray(splatData); + } else { + return SplatBuffer.generateFromUncompressedSplatArrays([splatData], minimumAlpha, 0, new THREE.Vector3()); + } +} + export class SplatLoader { static loadFromURL(fileName, onProgress, progressiveLoad, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, - optimizeSplatData, sectionSize, sceneCenter, blockSize, bucketSize) { + optimizeSplatData = true, sectionSize, sceneCenter, blockSize, bucketSize) { + + if (progressiveLoad) optimizeSplatData = false; const splatDataOffsetBytes = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes; const progressiveLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; @@ -32,7 +46,9 @@ export class SplatLoader { const localOnProgress = (percent, percentStr, chunk, fileSize) => { const loadComplete = percent >= 100; - if (!fileSize) progressiveLoad = false; + if (!fileSize) { + throw new DirectLoadError('Cannon directly load .splat because no file size info is available.'); + } if (!progressiveLoadBufferIn) { maxSplatCount = fileSize / SplatParser.RowSizeBytes; @@ -94,7 +110,9 @@ export class SplatLoader { progressiveLoadSplatBuffer = new SplatBuffer(progressiveLoadBufferOut, false); } progressiveLoadSplatBuffer.updateLoadedCounts(1, splatCount); - onProgressiveLoadSectionProgress(progressiveLoadSplatBuffer, loadComplete); + if (onProgressiveLoadSectionProgress) { + onProgressiveLoadSectionProgress(progressiveLoadSplatBuffer, loadComplete); + } } numBytesStreamed += progressiveLoadSectionSizeBytes; @@ -114,7 +132,7 @@ export class SplatLoader { }; if (onProgress) onProgress(0, '0%', LoaderStatus.Downloading); - return fetchWithProgress(fileName, localOnProgress, true).then(() => { + return fetchWithProgress(fileName, localOnProgress, false).then(() => { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); return loadPromise.promise.then((splatData) => { if (onProgress) onProgress(100, '100%', LoaderStatus.Done); @@ -122,14 +140,8 @@ export class SplatLoader { return splatData; } else { return delayedExecute(() => { - if (optimizeSplatData) { - const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, - sectionSize, sceneCenter, blockSize, - bucketSize); - return splatBufferGenerator.generateFromUncompressedSplatArray(splatData); - } else { - return SplatBuffer.generateFromUncompressedSplatArrays([splatData], minimumAlpha, 0, new THREE.Vector3()); - } + return finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, + sectionSize, sceneCenter, blockSize, bucketSize); }); } }); @@ -140,14 +152,8 @@ export class SplatLoader { sectionSize, sceneCenter, blockSize, bucketSize) { return delayedExecute(() => { const splatArray = SplatParser.parseStandardSplatToUncompressedSplatArray(splatFileData); - if (optimizeSplatData) { - const splatBufferGenerator = SplatBufferGenerator.getStandardGenerator(minimumAlpha, compressionLevel, - sectionSize, sceneCenter, blockSize, - bucketSize); - return splatBufferGenerator.generateFromUncompressedSplatArray(splatArray); - } else { - return SplatBuffer.generateFromUncompressedSplatArrays([splatArray], minimumAlpha, 0, new THREE.Vector3()); - } + return finalize(splatArray, optimizeSplatData, minimumAlpha, compressionLevel, + sectionSize, sceneCenter, blockSize, bucketSize); }); } From f15688cef6e399e15270e8b944da9dfc81d0abd1 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Mon, 2 Sep 2024 18:30:38 -0700 Subject: [PATCH 09/25] Improve memory efficiency of splat reorder & cleanup --- src/Viewer.js | 4 +- src/loaders/SplatPartitioner.js | 2 +- src/loaders/ksplat/KSplatLoader.js | 34 ++++++++-------- src/loaders/ply/PlyLoader.js | 57 ++++++++++++++------------ src/loaders/splat/SplatLoader.js | 65 ++++++++++++++++-------------- 5 files changed, 86 insertions(+), 76 deletions(-) diff --git a/src/Viewer.js b/src/Viewer.js index 3f217bf0..6ef2df62 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -20,7 +20,7 @@ import { VRButton } from './webxr/VRButton.js'; import { ARButton } from './webxr/ARButton.js'; import { delayedExecute, abortablePromiseWithExtractedComponents } from './Util.js'; import { LoaderStatus } from './loaders/LoaderStatus.js'; -import { DirectLoadError } from '../DirectLoadError.js'; +import { DirectLoadError } from './loaders/DirectLoadError.js'; import { RenderMode } from './RenderMode.js'; import { LogLevel } from './LogLevel.js'; import { SceneRevealMode } from './SceneRevealMode.js'; @@ -169,7 +169,7 @@ export class Viewer { if (options.optimizeSplatData === undefined || options.optimizeSplatData === null) { options.optimizeSplatData = true; } - this.optimizeSplatData = options.optimizeSplatData; + this.optimizeSplatData = options.optimizeSplatData || false; // When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and is used to // populate the data textures will be freed. This will reduces memory usage, but if that data needs to be modified diff --git a/src/loaders/SplatPartitioner.js b/src/loaders/SplatPartitioner.js index 36955796..f9c38eb5 100644 --- a/src/loaders/SplatPartitioner.js +++ b/src/loaders/SplatPartitioner.js @@ -32,7 +32,7 @@ export class SplatPartitioner { const sectionFilter = sectionFilters[s]; for (let i = 0; i < splatArray.splatCount; i++) { if (sectionFilter(i)) { - sectionSplats.addSplatFromArray(splatArray, i); + sectionSplats.addSplat(splatArray.splats[i]); } } newArrays.push(sectionSplats); diff --git a/src/loaders/ksplat/KSplatLoader.js b/src/loaders/ksplat/KSplatLoader.js index 8520a382..f09eee14 100644 --- a/src/loaders/ksplat/KSplatLoader.js +++ b/src/loaders/ksplat/KSplatLoader.js @@ -19,9 +19,9 @@ export class KSplatLoader { } }; - static loadFromURL(fileName, externalOnProgress, progressiveLoad, onSectionBuilt) { - let progressiveLoadBuffer; - let progressiveLoadSplatBuffer; + static loadFromURL(fileName, externalOnProgress, directLoad, onSectionBuilt) { + let directLoadBuffer; + let directLoadSplatBuffer; let headerBuffer; let header; @@ -43,7 +43,7 @@ export class KSplatLoader { let chunks = []; - const progressiveLoadPromise = nativePromiseWithExtractedComponents(); + const directLoadPromise = nativePromiseWithExtractedComponents(); const checkAndLoadHeader = () => { if (!headerLoaded && !headerLoading && numBytesLoaded >= SplatBuffer.HeaderSizeBytes) { @@ -91,12 +91,12 @@ export class KSplatLoader { } const totalStorageSizeBytes = SplatBuffer.HeaderSizeBytes + header.maxSectionCount * SplatBuffer.SectionHeaderSizeBytes + totalSectionStorageStorageByes; - if (!progressiveLoadBuffer) { - progressiveLoadBuffer = new ArrayBuffer(totalStorageSizeBytes); + if (!directLoadBuffer) { + directLoadBuffer = new ArrayBuffer(totalStorageSizeBytes); let offset = 0; for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; - new Uint8Array(progressiveLoadBuffer, offset, chunk.byteLength).set(new Uint8Array(chunk)); + new Uint8Array(directLoadBuffer, offset, chunk.byteLength).set(new Uint8Array(chunk)); offset += chunk.byteLength; } } @@ -133,7 +133,7 @@ export class KSplatLoader { numBytesProgressivelyLoaded += Constants.ProgressiveLoadSectionSize; loadComplete = numBytesProgressivelyLoaded >= totalBytesToDownload; - if (!progressiveLoadSplatBuffer) progressiveLoadSplatBuffer = new SplatBuffer(progressiveLoadBuffer, false); + if (!directLoadSplatBuffer) directLoadSplatBuffer = new SplatBuffer(directLoadBuffer, false); const baseDataOffset = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes * header.maxSectionCount; let sectionBase = 0; @@ -153,15 +153,15 @@ export class KSplatLoader { let loadedSplatsForSection = Math.floor(bytesPastSSectionSplatDataStart / bytesPerSplat); loadedSplatsForSection = Math.min(loadedSplatsForSection, sectionHeader.maxSplatCount); loadedSplatCount += loadedSplatsForSection; - progressiveLoadSplatBuffer.updateLoadedCounts(reachedSections, loadedSplatCount); - progressiveLoadSplatBuffer.updateSectionLoadedCounts(i, loadedSplatsForSection); + directLoadSplatBuffer.updateLoadedCounts(reachedSections, loadedSplatCount); + directLoadSplatBuffer.updateSectionLoadedCounts(i, loadedSplatsForSection); } else { break; } sectionBase += sectionHeader.storageSizeBytes; } - onSectionBuilt(progressiveLoadSplatBuffer, loadComplete); + onSectionBuilt(directLoadSplatBuffer, loadComplete); const percentComplete = numBytesProgressivelyLoaded / totalBytesToDownload * 100; const percentLabel = (percentComplete).toFixed(2) + '%'; @@ -169,7 +169,7 @@ export class KSplatLoader { if (externalOnProgress) externalOnProgress(percentComplete, percentLabel, LoaderStatus.Downloading); if (loadComplete) { - progressiveLoadPromise.resolve(progressiveLoadSplatBuffer); + directLoadPromise.resolve(directLoadSplatBuffer); } else { checkAndLoadSections(); } @@ -182,12 +182,12 @@ export class KSplatLoader { const localOnProgress = (percent, percentStr, chunk) => { if (chunk) { chunks.push(chunk); - if (progressiveLoadBuffer) { - new Uint8Array(progressiveLoadBuffer, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); + if (directLoadBuffer) { + new Uint8Array(directLoadBuffer, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); } numBytesLoaded += chunk.byteLength; } - if (progressiveLoad) { + if (directLoad) { checkAndLoadHeader(); checkAndLoadSectionHeaders(); checkAndLoadSections(); @@ -196,9 +196,9 @@ export class KSplatLoader { } }; - return fetchWithProgress(fileName, localOnProgress, !progressiveLoad).then((fullBuffer) => { + return fetchWithProgress(fileName, localOnProgress, !directLoad).then((fullBuffer) => { if (externalOnProgress) externalOnProgress(0, '0%', LoaderStatus.Processing); - const loadPromise = progressiveLoad ? progressiveLoadPromise.promise : KSplatLoader.loadFromFileData(fullBuffer); + const loadPromise = directLoad ? directLoadPromise.promise : KSplatLoader.loadFromFileData(fullBuffer); return loadPromise.then((splatBuffer) => { if (externalOnProgress) externalOnProgress(100, '100%', LoaderStatus.Done); return splatBuffer; diff --git a/src/loaders/ply/PlyLoader.js b/src/loaders/ply/PlyLoader.js index ebb80635..54185a21 100644 --- a/src/loaders/ply/PlyLoader.js +++ b/src/loaders/ply/PlyLoader.js @@ -42,18 +42,19 @@ function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, export class PlyLoader { - static loadFromURL(fileName, onProgress, progressiveLoad, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, + static loadFromURL(fileName, onProgress, directLoad, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, optimizeSplatData = true, outSphericalHarmonicsDegree = 0, sectionSize, sceneCenter, blockSize, bucketSize) { - if (progressiveLoad) optimizeSplatData = false; + const directLoadOriginalValue = directLoad; + if (optimizeSplatData) directLoad = false; - const progressiveLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; + const directLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; const splatDataOffsetBytes = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes; const sectionCount = 1; - let progressiveLoadBufferIn; - let progressiveLoadBufferOut; - let progressiveLoadSplatBuffer; + let directLoadBufferIn; + let directLoadBufferOut; + let directLoadSplatBuffer; let compressedPlyHeaderChunksBuffer; let maxSplatCount = 0; let splatCount = 0; @@ -103,15 +104,19 @@ export class PlyLoader { maxSplatCount = header.vertexElement.count; compressed = true; } else { - throw new DirectLoadError('PlyLoader.loadFromURL() -> Selected Ply format cannot be directly loaded.'); + if (directLoadOriginalValue) { + throw new DirectLoadError('PlyLoader.loadFromURL() -> Selected Ply format cannot be directly loaded.'); + } else { + directLoad = false; + } } outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; const splatBufferSizeBytes = splatDataOffsetBytes + shDescriptor.BytesPerSplat * maxSplatCount; - if (progressiveLoad) { - progressiveLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); + if (directLoad) { + directLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); SplatBuffer.writeHeaderToBuffer({ versionMajor: SplatBuffer.CurrentMajorVersion, versionMinor: SplatBuffer.CurrentMinorVersion, @@ -121,7 +126,7 @@ export class PlyLoader { splatCount: splatCount, compressionLevel: 0, sceneCenter: new THREE.Vector3() - }, progressiveLoadBufferOut); + }, directLoadBufferOut); } else { standardLoadUncompressedSplatArray = new UncompressedSplatArray(outSphericalHarmonicsDegree); } @@ -146,31 +151,31 @@ export class PlyLoader { if (chunks.length > 0) { - progressiveLoadBufferIn = storeChunksInBuffer(chunks, progressiveLoadBufferIn); + directLoadBufferIn = storeChunksInBuffer(chunks, directLoadBufferIn); const bytesLoadedSinceLastStreamedSection = numBytesDownloaded - numBytesStreamed; - if (bytesLoadedSinceLastStreamedSection > progressiveLoadSectionSizeBytes || loadComplete) { + if (bytesLoadedSinceLastStreamedSection > directLoadSectionSizeBytes || loadComplete) { const numBytesToProcess = numBytesDownloaded - numBytesParsed; const addedSplatCount = Math.floor(numBytesToProcess / header.bytesPerSplat); const numBytesToParse = addedSplatCount * header.bytesPerSplat; const numBytesLeftOver = numBytesToProcess - numBytesToParse; const newSplatCount = splatCount + addedSplatCount; const parsedDataViewOffset = numBytesParsed - chunks[0].startBytes; - const dataToParse = new DataView(progressiveLoadBufferIn, parsedDataViewOffset, numBytesToParse); + const dataToParse = new DataView(directLoadBufferIn, parsedDataViewOffset, numBytesToParse); const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; const outOffset = splatCount * shDescriptor.BytesPerSplat + splatDataOffsetBytes; - if (progressiveLoad) { + if (directLoad) { if (compressed) { PlayCanvasCompressedPlyParser.parseToUncompressedSplatBufferSection(header.chunkElement, header.vertexElement, 0, addedSplatCount - 1, splatCount, dataToParse, 0, - progressiveLoadBufferOut, outOffset); + directLoadBufferOut, outOffset); } else { inriaV1PlyParser.parseToUncompressedSplatBufferSection(header, 0, addedSplatCount - 1, dataToParse, - 0, progressiveLoadBufferOut, outOffset, + 0, directLoadBufferOut, outOffset, outSphericalHarmonicsDegree); } } else { @@ -189,8 +194,8 @@ export class PlyLoader { splatCount = newSplatCount; - if (progressiveLoad) { - if (!progressiveLoadSplatBuffer) { + if (directLoad) { + if (!directLoadSplatBuffer) { SplatBuffer.writeSectionHeaderToBuffer({ maxSplatCount: maxSplatCount, splatCount: splatCount, @@ -202,16 +207,16 @@ export class PlyLoader { fullBucketCount: 0, partiallyFilledBucketCount: 0, sphericalHarmonicsDegree: outSphericalHarmonicsDegree - }, 0, progressiveLoadBufferOut, SplatBuffer.HeaderSizeBytes); - progressiveLoadSplatBuffer = new SplatBuffer(progressiveLoadBufferOut, false); + }, 0, directLoadBufferOut, SplatBuffer.HeaderSizeBytes); + directLoadSplatBuffer = new SplatBuffer(directLoadBufferOut, false); } - progressiveLoadSplatBuffer.updateLoadedCounts(1, splatCount); + directLoadSplatBuffer.updateLoadedCounts(1, splatCount); if (onProgressiveLoadSectionProgress) { - onProgressiveLoadSectionProgress(progressiveLoadSplatBuffer, loadComplete); + onProgressiveLoadSectionProgress(directLoadSplatBuffer, loadComplete); } } - numBytesStreamed += progressiveLoadSectionSizeBytes; + numBytesStreamed += directLoadSectionSizeBytes; numBytesParsed += numBytesToParse; if (numBytesLeftOver === 0) { @@ -231,8 +236,8 @@ export class PlyLoader { } if (loadComplete) { - if (progressiveLoad) { - loadPromise.resolve(progressiveLoadSplatBuffer); + if (directLoad) { + loadPromise.resolve(directLoadSplatBuffer); } else { loadPromise.resolve(standardLoadUncompressedSplatArray); } @@ -247,7 +252,7 @@ export class PlyLoader { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); return loadPromise.promise.then((splatData) => { if (onProgress) onProgress(100, '100%', LoaderStatus.Done); - if (progressiveLoad) { + if (directLoad) { return splatData; } else { return delayedExecute(() => { diff --git a/src/loaders/splat/SplatLoader.js b/src/loaders/splat/SplatLoader.js index 70305ad2..31313ba7 100644 --- a/src/loaders/splat/SplatLoader.js +++ b/src/loaders/splat/SplatLoader.js @@ -21,18 +21,19 @@ function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, export class SplatLoader { - static loadFromURL(fileName, onProgress, progressiveLoad, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, + static loadFromURL(fileName, onProgress, directLoad, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, optimizeSplatData = true, sectionSize, sceneCenter, blockSize, bucketSize) { - if (progressiveLoad) optimizeSplatData = false; + const directLoadOriginalValue = directLoad; + if (optimizeSplatData) directLoad = false; const splatDataOffsetBytes = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes; - const progressiveLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; + const directLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; const sectionCount = 1; - let progressiveLoadBufferIn; - let progressiveLoadBufferOut; - let progressiveLoadSplatBuffer; + let directLoadBufferIn; + let directLoadBufferOut; + let directLoadSplatBuffer; let maxSplatCount = 0; let splatCount = 0; @@ -47,17 +48,21 @@ export class SplatLoader { const localOnProgress = (percent, percentStr, chunk, fileSize) => { const loadComplete = percent >= 100; if (!fileSize) { - throw new DirectLoadError('Cannon directly load .splat because no file size info is available.'); + if (directLoadOriginalValue) { + throw new DirectLoadError('Cannon directly load .splat because no file size info is available.'); + } else { + directLoad = false; + } } - if (!progressiveLoadBufferIn) { + if (!directLoadBufferIn) { maxSplatCount = fileSize / SplatParser.RowSizeBytes; - progressiveLoadBufferIn = new ArrayBuffer(fileSize); + directLoadBufferIn = new ArrayBuffer(fileSize); const bytesPerSplat = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[0].BytesPerSplat; const splatBufferSizeBytes = splatDataOffsetBytes + bytesPerSplat * maxSplatCount; - if (progressiveLoad) { - progressiveLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); + if (directLoad) { + directLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); SplatBuffer.writeHeaderToBuffer({ versionMajor: SplatBuffer.CurrentMajorVersion, versionMinor: SplatBuffer.CurrentMinorVersion, @@ -67,7 +72,7 @@ export class SplatLoader { splatCount: splatCount, compressionLevel: 0, sceneCenter: new THREE.Vector3() - }, progressiveLoadBufferOut); + }, directLoadBufferOut); } else { standardLoadUncompressedSplatArray = new UncompressedSplatArray(0); } @@ -75,27 +80,27 @@ export class SplatLoader { if (chunk) { chunks.push(chunk); - new Uint8Array(progressiveLoadBufferIn, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); + new Uint8Array(directLoadBufferIn, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); numBytesLoaded += chunk.byteLength; const bytesLoadedSinceLastSection = numBytesLoaded - numBytesStreamed; - if (bytesLoadedSinceLastSection > progressiveLoadSectionSizeBytes || loadComplete) { - const bytesToUpdate = loadComplete ? bytesLoadedSinceLastSection : progressiveLoadSectionSizeBytes; + if (bytesLoadedSinceLastSection > directLoadSectionSizeBytes || loadComplete) { + const bytesToUpdate = loadComplete ? bytesLoadedSinceLastSection : directLoadSectionSizeBytes; const addedSplatCount = bytesToUpdate / SplatParser.RowSizeBytes; const newSplatCount = splatCount + addedSplatCount; - if (progressiveLoad) { - SplatParser.parseToUncompressedSplatBufferSection(splatCount, newSplatCount - 1, progressiveLoadBufferIn, 0, - progressiveLoadBufferOut, splatDataOffsetBytes); + if (directLoad) { + SplatParser.parseToUncompressedSplatBufferSection(splatCount, newSplatCount - 1, directLoadBufferIn, 0, + directLoadBufferOut, splatDataOffsetBytes); } else { - SplatParser.parseToUncompressedSplatArraySection(splatCount, newSplatCount - 1, progressiveLoadBufferIn, 0, + SplatParser.parseToUncompressedSplatArraySection(splatCount, newSplatCount - 1, directLoadBufferIn, 0, standardLoadUncompressedSplatArray); } splatCount = newSplatCount; - if (progressiveLoad) { - if (!progressiveLoadSplatBuffer) { + if (directLoad) { + if (!directLoadSplatBuffer) { SplatBuffer.writeSectionHeaderToBuffer({ maxSplatCount: maxSplatCount, splatCount: splatCount, @@ -106,29 +111,29 @@ export class SplatLoader { storageSizeBytes: 0, fullBucketCount: 0, partiallyFilledBucketCount: 0 - }, 0, progressiveLoadBufferOut, SplatBuffer.HeaderSizeBytes); - progressiveLoadSplatBuffer = new SplatBuffer(progressiveLoadBufferOut, false); + }, 0, directLoadBufferOut, SplatBuffer.HeaderSizeBytes); + directLoadSplatBuffer = new SplatBuffer(directLoadBufferOut, false); } - progressiveLoadSplatBuffer.updateLoadedCounts(1, splatCount); + directLoadSplatBuffer.updateLoadedCounts(1, splatCount); if (onProgressiveLoadSectionProgress) { - onProgressiveLoadSectionProgress(progressiveLoadSplatBuffer, loadComplete); + onProgressiveLoadSectionProgress(directLoadSplatBuffer, loadComplete); } } - numBytesStreamed += progressiveLoadSectionSizeBytes; + numBytesStreamed += directLoadSectionSizeBytes; } } if (loadComplete) { - if (progressiveLoad) { - loadPromise.resolve(progressiveLoadSplatBuffer); + if (directLoad) { + loadPromise.resolve(directLoadSplatBuffer); } else { loadPromise.resolve(standardLoadUncompressedSplatArray); } } if (onProgress) onProgress(percent, percentStr, LoaderStatus.Downloading); - return progressiveLoad; + return directLoad; }; if (onProgress) onProgress(0, '0%', LoaderStatus.Downloading); @@ -136,7 +141,7 @@ export class SplatLoader { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); return loadPromise.promise.then((splatData) => { if (onProgress) onProgress(100, '100%', LoaderStatus.Done); - if (progressiveLoad) { + if (directLoad) { return splatData; } else { return delayedExecute(() => { From b42c6f7ea2d89656d7d0ef07c1b6a1a9d2388adf Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Mon, 2 Sep 2024 19:31:10 -0700 Subject: [PATCH 10/25] Added SplatMesh.computeBoundingBox() --- src/splatmesh/SplatMesh.js | 40 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/splatmesh/SplatMesh.js b/src/splatmesh/SplatMesh.js index bea7ed2d..da5874ca 100644 --- a/src/splatmesh/SplatMesh.js +++ b/src/splatmesh/SplatMesh.js @@ -1846,7 +1846,7 @@ export class SplatMesh extends THREE.Mesh { */ fillSplatDataArrays(covariances, scales, rotations, centers, colors, sphericalHarmonics, applySceneTransform, covarianceCompressionLevel = 0, scaleRotationCompressionLevel = 0, sphericalHarmonicsCompressionLevel = 1, - srcStart, srcEnd, destStart = 0) { + srcStart, srcEnd, destStart = 0, sceneIndex) { const scaleOverride = new THREE.Vector3(); scaleOverride.x = undefined; scaleOverride.y = undefined; @@ -1857,7 +1857,13 @@ export class SplatMesh extends THREE.Mesh { } const tempTransform = new THREE.Matrix4(); - for (let i = 0; i < this.scenes.length; i++) { + let startSceneIndex = 0; + let endSceneIndex = this.scenes.length - 1; + if (sceneIndex !== undefined && sceneIndex !== null && sceneIndex >= 0 && sceneIndex <= this.scenes.length) { + startSceneIndex = sceneIndex; + endSceneIndex = sceneIndex; + } + for (let i = startSceneIndex; i <= endSceneIndex; i++) { if (applySceneTransform === undefined || applySceneTransform === null) { applySceneTransform = this.dynamicMode ? false : true; } @@ -2050,4 +2056,34 @@ export class SplatMesh extends THREE.Mesh { } return intMatrixArray; } + + computeBoundingBox(sceneIndex) { + let splatCount = this.getSplatCount(); + if (sceneIndex !== undefined && sceneIndex !== null) { + if (sceneIndex < 0 || sceneIndex >= this.scenes.length) { + throw new Error('SplatMesh::computeBoundingBox() -> Invalid scene index.'); + } + splatCount = this.scenes[sceneIndex].splatBuffer.getSplatCount(); + } + + const floatCenters = new Float32Array(splatCount * 3); + this.fillSplatDataArrays(null, null, null, floatCenters, null, null, false, undefined, undefined, undefined, undefined, sceneIndex); + + const min = new THREE.Vector3(); + const max = new THREE.Vector3(); + for (let i = 0; i < splatCount; i++) { + const offset = i * 3; + const x = floatCenters[offset]; + const y = floatCenters[offset + 1]; + const z = floatCenters[offset + 2]; + if (i === 0 || x < min.x) min.x = x; + if (i === 0 || y < min.y) min.y = y; + if (i === 0 || z < min.z) min.z = z; + if (i === 0 || x > max.x) max.x = x; + if (i === 0 || y > max.y) max.y = y; + if (i === 0 || z > max.z) max.z = z; + } + + return new THREE.Box3(min, max); + } } From 3ff35c2eb6e39ace27f6815beecbb7127e0ba559 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Mon, 2 Sep 2024 19:50:34 -0700 Subject: [PATCH 11/25] Add 'applySceneTransforms' parameter to SplatMesh.computeBoundingBox() --- src/splatmesh/SplatMesh.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/splatmesh/SplatMesh.js b/src/splatmesh/SplatMesh.js index da5874ca..014709c1 100644 --- a/src/splatmesh/SplatMesh.js +++ b/src/splatmesh/SplatMesh.js @@ -2057,7 +2057,7 @@ export class SplatMesh extends THREE.Mesh { return intMatrixArray; } - computeBoundingBox(sceneIndex) { + computeBoundingBox(applySceneTransforms = false, sceneIndex) { let splatCount = this.getSplatCount(); if (sceneIndex !== undefined && sceneIndex !== null) { if (sceneIndex < 0 || sceneIndex >= this.scenes.length) { @@ -2067,7 +2067,8 @@ export class SplatMesh extends THREE.Mesh { } const floatCenters = new Float32Array(splatCount * 3); - this.fillSplatDataArrays(null, null, null, floatCenters, null, null, false, undefined, undefined, undefined, undefined, sceneIndex); + this.fillSplatDataArrays(null, null, null, floatCenters, null, null, applySceneTransforms, + undefined, undefined, undefined, undefined, sceneIndex); const min = new THREE.Vector3(); const max = new THREE.Vector3(); From 8249f1ffd1125804f1ef356cf5b1bdc3da70fba1 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Mon, 2 Sep 2024 20:19:27 -0700 Subject: [PATCH 12/25] Tweaks to iOS WASM config --- src/worker/SortWorker.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/worker/SortWorker.js b/src/worker/SortWorker.js index 9b73a0f1..3f6cb48f 100644 --- a/src/worker/SortWorker.js +++ b/src/worker/SortWorker.js @@ -209,16 +209,16 @@ export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, let sourceWasm = SorterWasm; // iOS makes choosing the right WebAssembly configuration tricky :( - let iOSSemVer = isIOS() ? getIOSSemever() : null; + const iOSSemVer = isIOS() ? getIOSSemever() : null; if (!enableSIMDInSort && !useSharedMemory) { sourceWasm = SorterWasmNoSIMD; - if (iOSSemVer && iOSSemVer.major < 16 && iOSSemVer.minor < 4) { + if (iOSSemVer && iOSSemVer.major <= 16 && iOSSemVer.minor < 4) { sourceWasm = SorterWasmNoSIMDNonShared; } } else if (!enableSIMDInSort) { sourceWasm = SorterWasmNoSIMD; } else if (!useSharedMemory) { - if (iOSSemVer && iOSSemVer.major < 16 && iOSSemVer.minor < 4) { + if (iOSSemVer && iOSSemVer.major <= 16 && iOSSemVer.minor < 4) { sourceWasm = SorterWasmNonShared; } } From 9e2f2b0e57e11c85fc54c647e40d76d46f142999 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Tue, 3 Sep 2024 08:05:29 -0700 Subject: [PATCH 13/25] All paths in runSplatSort() return promise --- src/Viewer.js | 2 +- src/worker/SortWorker.js | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Viewer.js b/src/Viewer.js index 6ef2df62..a566c730 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -1824,7 +1824,7 @@ export class Viewer { if (this.sortRunning) return Promise.resolve(true); if (this.splatMesh.getSplatCount() <= 0) { this.splatRenderCount = 0; - return false; + return Promise.resolve(false); } let angleDiff = 0; diff --git a/src/worker/SortWorker.js b/src/worker/SortWorker.js index 3f6cb48f..990b4438 100644 --- a/src/worker/SortWorker.js +++ b/src/worker/SortWorker.js @@ -212,12 +212,15 @@ export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, const iOSSemVer = isIOS() ? getIOSSemever() : null; if (!enableSIMDInSort && !useSharedMemory) { sourceWasm = SorterWasmNoSIMD; + // Testing on various devices has shown that even when shared memory is disabled, the WASM module with shared + // memory can still be used most of the time -- the exception seems to be iOS devices below 16.4 if (iOSSemVer && iOSSemVer.major <= 16 && iOSSemVer.minor < 4) { sourceWasm = SorterWasmNoSIMDNonShared; } } else if (!enableSIMDInSort) { sourceWasm = SorterWasmNoSIMD; } else if (!useSharedMemory) { + // Same issue with shared memory as above on iOS devices if (iOSSemVer && iOSSemVer.major <= 16 && iOSSemVer.minor < 4) { sourceWasm = SorterWasmNonShared; } From b74b7d76d0a8770c321ede7f8694af17037b112b Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Wed, 4 Sep 2024 06:59:54 -0700 Subject: [PATCH 14/25] Don't run shouldRender() if dispose has occurred --- src/DropInViewer.js | 4 ++-- src/Viewer.js | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/DropInViewer.js b/src/DropInViewer.js index 764b560e..5db045f5 100644 --- a/src/DropInViewer.js +++ b/src/DropInViewer.js @@ -116,8 +116,8 @@ export class DropInViewer extends THREE.Group { this.viewer.setActiveSphericalHarmonicsDegrees(activeSphericalHarmonicsDegrees); } - dispose() { - return this.viewer.dispose(); + async dispose() { + return await this.viewer.dispose(); } static onBeforeRender(viewer, renderer, threeScene, camera) { diff --git a/src/Viewer.js b/src/Viewer.js index a566c730..37570c83 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -158,7 +158,7 @@ export class Viewer { if (options.enableSIMDInSort === undefined || options.enableSIMDInSort === null) options.enableSIMDInSort = true; this.enableSIMDInSort = options.enableSIMDInSort; - // Level to compress PLY files when loading them for direct rendering (not exporting to .ksplat) + // Level to compress non KSPLAT files when loading them for direct rendering if (options.inMemoryCompressionLevel === undefined || options.inMemoryCompressionLevel === null) { options.inMemoryCompressionLevel = 0; } @@ -1424,7 +1424,9 @@ export class Viewer { */ stop() { if (this.selfDrivenMode && this.selfDrivenModeRunning) { - if (!this.webXRMode) { + if (this.webXRMode) { + this.renderer.setAnimationLoop(null); + } else { cancelAnimationFrame(this.requestFrameId); } this.selfDrivenModeRunning = false; @@ -1534,6 +1536,9 @@ export class Viewer { const changeEpsilon = 0.0001; return function() { + if (!this.initialized || !this.splatRenderReady) return false; + if (this.isDisposingOrDisposed()) return false; + let shouldRender = false; let cameraChanged = false; if (this.camera) { @@ -1566,6 +1571,7 @@ export class Viewer { return function() { if (!this.initialized || !this.splatRenderReady) return; + if (this.isDisposingOrDisposed()) return; const hasRenderables = (threeScene) => { for (let child of threeScene.children) { @@ -1590,7 +1596,10 @@ export class Viewer { update(renderer, camera) { if (this.dropInMode) this.updateForDropInMode(renderer, camera); + if (!this.initialized || !this.splatRenderReady) return; + if (this.isDisposingOrDisposed()) return; + if (this.controls) { this.controls.update(); if (this.camera.isOrthographicCamera && !this.usingExternalCamera) { From 9680ca0eb395263ebcf31792833e4d0cdc8e2e3f Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sat, 7 Sep 2024 19:42:21 -0700 Subject: [PATCH 15/25] Add viewer option to set splat sort depth map range --- src/Constants.js | 2 +- src/Viewer.js | 5 ++++- src/worker/SortWorker.js | 15 +++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Constants.js b/src/Constants.js index cec02275..e293ba2e 100644 --- a/src/Constants.js +++ b/src/Constants.js @@ -1,6 +1,6 @@ export class Constants { - static DepthMapRange = 1 << 16; + static DefaultSortDepthMapRange = 1 << 16; static MemoryPageSize = 65536; static BytesPerFloat = 4; static BytesPerInt = 4; diff --git a/src/Viewer.js b/src/Viewer.js index 37570c83..40dd5e75 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -200,6 +200,9 @@ export class Viewer { // Customize the speed at which the scene is revealed this.sceneFadeInRateMultiplier = options.sceneFadeInRateMultiplier || 1.0; + // Set the range for the counting sort used to sort the splats + this.splatSortDepthMapRange = options.splatSortDepthMapRange || Constants.DefaultSortDepthMapRange; + this.onSplatMeshChangedCallback = null; this.createSplatMesh(); @@ -1226,7 +1229,7 @@ export class Viewer { const splatCount = splatMesh.getSplatCount(); const maxSplatCount = splatMesh.getMaxSplatCount(); this.sortWorker = createSortWorker(maxSplatCount, this.sharedMemoryForWorkers, this.enableSIMDInSort, - this.integerBasedSort, this.splatMesh.dynamicMode); + this.integerBasedSort, this.splatMesh.dynamicMode, this.splatSortDepthMapRange); this.sortWorker.onmessage = (e) => { if (e.data.sortDone) { this.sortRunning = false; diff --git a/src/worker/SortWorker.js b/src/worker/SortWorker.js index 990b4438..8d3f1a7b 100644 --- a/src/worker/SortWorker.js +++ b/src/worker/SortWorker.js @@ -24,6 +24,7 @@ function sortWorker(self) { let modelViewProjOffset; let countsZero; let sortedIndexesOut; + let depthMapRange; let Constants; @@ -49,12 +50,12 @@ function sortWorker(self) { } } - if (!countsZero) countsZero = new Uint32Array(Constants.DepthMapRange); + if (!countsZero) countsZero = new Uint32Array(depthMapRange); new Float32Array(wasmMemory, modelViewProjOffset, 16).set(modelViewProj); - new Uint32Array(wasmMemory, frequenciesOffset, Constants.DepthMapRange).set(countsZero); + new Uint32Array(wasmMemory, frequenciesOffset, depthMapRange).set(countsZero); wasmInstance.exports.sortIndexes(indexesToSortOffset, centersOffset, precomputedDistancesOffset, mappedDistancesOffset, frequenciesOffset, modelViewProjOffset, - sortedIndexesOffset, sceneIndexesOffset, transformsOffset, Constants.DepthMapRange, + sortedIndexesOffset, sceneIndexesOffset, transformsOffset, depthMapRange, splatSortCount, splatRenderCount, splatCount, usePrecomputedDistances, integerBasedSort, dynamicMode); @@ -120,6 +121,7 @@ function sortWorker(self) { useSharedMemory = e.data.init.useSharedMemory; integerBasedSort = e.data.init.integerBasedSort; dynamicMode = e.data.init.dynamicMode; + depthMapRange = e.data.init.depthMapRange; const CENTERS_BYTES_PER_ENTRY = integerBasedSort ? (Constants.BytesPerInt * 4) : (Constants.BytesPerFloat * 4); @@ -133,7 +135,7 @@ function sortWorker(self) { (splatCount * Constants.BytesPerInt) : (splatCount * Constants.BytesPerFloat); const memoryRequiredForMappedDistances = splatCount * Constants.BytesPerInt; const memoryRequiredForSortedIndexes = splatCount * Constants.BytesPerInt; - const memoryRequiredForIntermediateSortBuffers = Constants.DepthMapRange * Constants.BytesPerInt * 2; + const memoryRequiredForIntermediateSortBuffers = depthMapRange * Constants.BytesPerInt * 2; const memoryRequiredforTransformIndexes = dynamicMode ? (splatCount * Constants.BytesPerInt) : 0; const memoryRequiredforTransforms = dynamicMode ? (Constants.MaxScenes * matrixSize) : 0; const extraMemory = Constants.MemoryPageSize * 32; @@ -197,7 +199,8 @@ function sortWorker(self) { }; } -export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, integerBasedSort, dynamicMode) { +export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, + integerBasedSort, dynamicMode, depthMapRange = Constants.DefaultSortDepthMapRange) { const worker = new Worker( URL.createObjectURL( new Blob(['(', sortWorker.toString(), ')(self)'], { @@ -239,11 +242,11 @@ export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, 'useSharedMemory': useSharedMemory, 'integerBasedSort': integerBasedSort, 'dynamicMode': dynamicMode, + 'depthMapRange': depthMapRange, // Super hacky 'Constants': { 'BytesPerFloat': Constants.BytesPerFloat, 'BytesPerInt': Constants.BytesPerInt, - 'DepthMapRange': Constants.DepthMapRange, 'MemoryPageSize': Constants.MemoryPageSize, 'MaxScenes': Constants.MaxScenes } From 33dcb3560231da8bd6d1fd05d5f4980565755a30 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sat, 7 Sep 2024 20:45:57 -0700 Subject: [PATCH 16/25] Add README entry for 'splatSortDepthMapRange' --- README.md | 1 + src/Constants.js | 2 +- src/Viewer.js | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d7af3ee7..d0aaebab 100644 --- a/README.md +++ b/README.md @@ -333,6 +333,7 @@ Advanced `Viewer` parameters | `freeIntermediateSplatData` | When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and used to populate data textures will be freed. This will reduces memory usage, but if that data needs to be modified it will need to be re-populated from the splat buffer(s). Defaults to `false`. | `splatRenderMode` | Determine which splat rendering mode to enable. Valid values are defined in the `SplatRenderMode` enum: `ThreeD` and `TwoD`. `ThreeD` is the original/traditional mode and `TwoD` is the new mode described here: https://surfsplatting.github.io/ | `sceneFadeInRateMultiplier` | Customize the speed at which the scene is revealed. Default is 1.0. +| `splatSortDepthMapRange` | Specify the range for the depth map for the counting sort used to sort the splats. Defaults to 65536.
### Creating KSPLAT files diff --git a/src/Constants.js b/src/Constants.js index e293ba2e..fe192f8c 100644 --- a/src/Constants.js +++ b/src/Constants.js @@ -1,6 +1,6 @@ export class Constants { - static DefaultSortDepthMapRange = 1 << 16; + static DefaultSortDepthMapRange = 65536; static MemoryPageSize = 65536; static BytesPerFloat = 4; static BytesPerInt = 4; diff --git a/src/Viewer.js b/src/Viewer.js index 40dd5e75..55ad9e6a 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -200,7 +200,7 @@ export class Viewer { // Customize the speed at which the scene is revealed this.sceneFadeInRateMultiplier = options.sceneFadeInRateMultiplier || 1.0; - // Set the range for the counting sort used to sort the splats + // Set the range for the depth map for the counting sort used to sort the splats this.splatSortDepthMapRange = options.splatSortDepthMapRange || Constants.DefaultSortDepthMapRange; this.onSplatMeshChangedCallback = null; From 6e43cdf64b53c22d15e7dc01575f2a27601f044a Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sun, 8 Sep 2024 11:26:50 -0700 Subject: [PATCH 17/25] Change 'splatSortDepthMapRange' parameter to 'splatSortDistanceMapPrecision' --- README.md | 2 +- src/Constants.js | 2 +- src/Viewer.js | 7 ++++--- src/worker/SortWorker.js | 18 +++++++++--------- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d0aaebab..5b9c4b6d 100644 --- a/README.md +++ b/README.md @@ -333,7 +333,7 @@ Advanced `Viewer` parameters | `freeIntermediateSplatData` | When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and used to populate data textures will be freed. This will reduces memory usage, but if that data needs to be modified it will need to be re-populated from the splat buffer(s). Defaults to `false`. | `splatRenderMode` | Determine which splat rendering mode to enable. Valid values are defined in the `SplatRenderMode` enum: `ThreeD` and `TwoD`. `ThreeD` is the original/traditional mode and `TwoD` is the new mode described here: https://surfsplatting.github.io/ | `sceneFadeInRateMultiplier` | Customize the speed at which the scene is revealed. Default is 1.0. -| `splatSortDepthMapRange` | Specify the range for the depth map for the counting sort used to sort the splats. Defaults to 65536. +| `splatSortDistanceMapPrecision` | Specify the precision for the distance map used in the splat sort algorithm. Defaults to 16.
### Creating KSPLAT files diff --git a/src/Constants.js b/src/Constants.js index fe192f8c..1694d2eb 100644 --- a/src/Constants.js +++ b/src/Constants.js @@ -1,6 +1,6 @@ export class Constants { - static DefaultSortDepthMapRange = 65536; + static DefaultSplatSortDistanceMapPrecision = 16; static MemoryPageSize = 65536; static BytesPerFloat = 4; static BytesPerInt = 4; diff --git a/src/Viewer.js b/src/Viewer.js index 55ad9e6a..c547d11c 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -12,7 +12,7 @@ import { Raycaster } from './raycaster/Raycaster.js'; import { SplatMesh } from './splatmesh/SplatMesh.js'; import { createSortWorker } from './worker/SortWorker.js'; import { Constants } from './Constants.js'; -import { getCurrentTime, isIOS, getIOSSemever } from './Util.js'; +import { getCurrentTime, isIOS, getIOSSemever, clamp } from './Util.js'; import { AbortablePromise, AbortedPromiseError } from './AbortablePromise.js'; import { SceneFormat } from './loaders/SceneFormat.js'; import { WebXRMode } from './webxr/WebXRMode.js'; @@ -201,7 +201,8 @@ export class Viewer { this.sceneFadeInRateMultiplier = options.sceneFadeInRateMultiplier || 1.0; // Set the range for the depth map for the counting sort used to sort the splats - this.splatSortDepthMapRange = options.splatSortDepthMapRange || Constants.DefaultSortDepthMapRange; + this.splatSortDistanceMapPrecision = options.splatSortDistanceMapPrecision || Constants.DefaultSplatSortDistanceMapPrecision; + this.splatSortDistanceMapPrecision = clamp(this.splatSortDistanceMapPrecision, 10, 24); this.onSplatMeshChangedCallback = null; this.createSplatMesh(); @@ -1229,7 +1230,7 @@ export class Viewer { const splatCount = splatMesh.getSplatCount(); const maxSplatCount = splatMesh.getMaxSplatCount(); this.sortWorker = createSortWorker(maxSplatCount, this.sharedMemoryForWorkers, this.enableSIMDInSort, - this.integerBasedSort, this.splatMesh.dynamicMode, this.splatSortDepthMapRange); + this.integerBasedSort, this.splatMesh.dynamicMode, this.splatSortDistanceMapPrecision); this.sortWorker.onmessage = (e) => { if (e.data.sortDone) { this.sortRunning = false; diff --git a/src/worker/SortWorker.js b/src/worker/SortWorker.js index 8d3f1a7b..2ff4704c 100644 --- a/src/worker/SortWorker.js +++ b/src/worker/SortWorker.js @@ -24,7 +24,7 @@ function sortWorker(self) { let modelViewProjOffset; let countsZero; let sortedIndexesOut; - let depthMapRange; + let distanceMapRange; let Constants; @@ -50,12 +50,12 @@ function sortWorker(self) { } } - if (!countsZero) countsZero = new Uint32Array(depthMapRange); + if (!countsZero) countsZero = new Uint32Array(distanceMapRange); new Float32Array(wasmMemory, modelViewProjOffset, 16).set(modelViewProj); - new Uint32Array(wasmMemory, frequenciesOffset, depthMapRange).set(countsZero); + new Uint32Array(wasmMemory, frequenciesOffset, distanceMapRange).set(countsZero); wasmInstance.exports.sortIndexes(indexesToSortOffset, centersOffset, precomputedDistancesOffset, mappedDistancesOffset, frequenciesOffset, modelViewProjOffset, - sortedIndexesOffset, sceneIndexesOffset, transformsOffset, depthMapRange, + sortedIndexesOffset, sceneIndexesOffset, transformsOffset, distanceMapRange, splatSortCount, splatRenderCount, splatCount, usePrecomputedDistances, integerBasedSort, dynamicMode); @@ -121,7 +121,7 @@ function sortWorker(self) { useSharedMemory = e.data.init.useSharedMemory; integerBasedSort = e.data.init.integerBasedSort; dynamicMode = e.data.init.dynamicMode; - depthMapRange = e.data.init.depthMapRange; + distanceMapRange = e.data.init.distanceMapRange; const CENTERS_BYTES_PER_ENTRY = integerBasedSort ? (Constants.BytesPerInt * 4) : (Constants.BytesPerFloat * 4); @@ -135,7 +135,7 @@ function sortWorker(self) { (splatCount * Constants.BytesPerInt) : (splatCount * Constants.BytesPerFloat); const memoryRequiredForMappedDistances = splatCount * Constants.BytesPerInt; const memoryRequiredForSortedIndexes = splatCount * Constants.BytesPerInt; - const memoryRequiredForIntermediateSortBuffers = depthMapRange * Constants.BytesPerInt * 2; + const memoryRequiredForIntermediateSortBuffers = distanceMapRange * Constants.BytesPerInt * 2; const memoryRequiredforTransformIndexes = dynamicMode ? (splatCount * Constants.BytesPerInt) : 0; const memoryRequiredforTransforms = dynamicMode ? (Constants.MaxScenes * matrixSize) : 0; const extraMemory = Constants.MemoryPageSize * 32; @@ -199,8 +199,8 @@ function sortWorker(self) { }; } -export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, - integerBasedSort, dynamicMode, depthMapRange = Constants.DefaultSortDepthMapRange) { +export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, integerBasedSort, dynamicMode, + splatSortDistanceMapPrecision = Constants.DefaultSplatSortDistanceMapPrecision) { const worker = new Worker( URL.createObjectURL( new Blob(['(', sortWorker.toString(), ')(self)'], { @@ -242,7 +242,7 @@ export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, 'useSharedMemory': useSharedMemory, 'integerBasedSort': integerBasedSort, 'dynamicMode': dynamicMode, - 'depthMapRange': depthMapRange, + 'distanceMapRange': 1 << splatSortDistanceMapPrecision, // Super hacky 'Constants': { 'BytesPerFloat': Constants.BytesPerFloat, From 8b9d92ce6763f95fd7c34a505c4d59a4d24beb0d Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Tue, 10 Sep 2024 13:14:31 -0700 Subject: [PATCH 18/25] Update README --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5b9c4b6d..ed58e5ea 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ When I started, web-based viewers were already available -- A WebGL-based viewer - Sub-optimal performance on mobile devices - Custom `.ksplat` file format still needs work, especially around compression - The default, integer based splat sort does not work well for larger scenes. In that case a value of `false` for the `integerBasedSort` viewer parameter can force a slower, floating-point based sort +- The default precision (16-bit) for the distance map in the splat sort may not work well for larger scenes, or scenes with a dense splat arrangement. For those scenes the viewer parameter `splatSortDistanceMapPrecision` can be used to adjust that value. Larger precision values will result in reduced performance, but often can alleviate visual artifacts that arise when the precision is too low. ## Limitations @@ -317,6 +318,7 @@ Advanced `Viewer` parameters | `enableSIMDInSort` | Enable the usage of SIMD WebAssembly instructions for the splat sort. Default is `true`. | `sharedMemoryForWorkers` | Tells the viewer to use shared memory via a `SharedArrayBuffer` to transfer data to and from the sorting web worker. If set to `false`, it is recommended that `gpuAcceleratedSort` be set to `false` as well. Defaults to `true`. | `integerBasedSort` | Tells the sorting web worker to use the integer versions of relevant data to compute the distance of splats from the camera. Since integer arithmetic is faster than floating point, this reduces sort time. However it can result in integer overflows in larger scenes so it should only be used for small scenes. Defaults to `true`. +| `splatSortDistanceMapPrecision` | Specify the precision for the distance map used in the splat sort algorithm. Defaults to 16 (16-bit). A lower precision is faster, but may result in visual artifacts in larger or denser scenes. | `halfPrecisionCovariancesOnGPU` | Tells the viewer to use 16-bit floating point values when storing splat covariance data in textures, instead of 32-bit. Defaults to `false`. | `dynamicScene` | Tells the viewer to not make any optimizations that depend on the scene being static. Additionally all splat data retrieved from the viewer's splat mesh will not have their respective scene transform applied to them by default. | `webXRMode` | Tells the viewer whether or not to enable built-in Web VR or Web AR. Valid values are defined in the `WebXRMode` enum: `None`, `VR`, and `AR`. Defaults to `None`. @@ -333,7 +335,6 @@ Advanced `Viewer` parameters | `freeIntermediateSplatData` | When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and used to populate data textures will be freed. This will reduces memory usage, but if that data needs to be modified it will need to be re-populated from the splat buffer(s). Defaults to `false`. | `splatRenderMode` | Determine which splat rendering mode to enable. Valid values are defined in the `SplatRenderMode` enum: `ThreeD` and `TwoD`. `ThreeD` is the original/traditional mode and `TwoD` is the new mode described here: https://surfsplatting.github.io/ | `sceneFadeInRateMultiplier` | Customize the speed at which the scene is revealed. Default is 1.0. -| `splatSortDistanceMapPrecision` | Specify the precision for the distance map used in the splat sort algorithm. Defaults to 16.
### Creating KSPLAT files @@ -352,7 +353,7 @@ GaussianSplats3D.PlyLoader.loadFromURL('', minimumAlpha, compressionLevel, optimizeSplatData, - outSphericalHarmonicsDegree) + sphericalHarmonicsDegree) .then((splatBuffer) => { GaussianSplats3D.KSplatLoader.downloadFile(splatBuffer, 'converted_file.ksplat'); }); From cf7b2e6cfb11b438361cdd71de846d22c3f5ecba Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Wed, 11 Sep 2024 18:41:56 -0700 Subject: [PATCH 19/25] Dispose both perspective and orthographic controls --- src/Viewer.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Viewer.js b/src/Viewer.js index c547d11c..b9a76b0e 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -1459,10 +1459,15 @@ export class Viewer { this.disposing = true; this.disposePromise = Promise.all(waitPromises).finally(() => { this.stop(); - if (this.controls) { - this.controls.dispose(); - this.controls = null; + if (this.orthographicControls) { + this.orthographicControls.dispose(); + this.orthographicControls = null; } + if (this.perspectiveControls) { + this.perspectiveControls.dispose(); + this.perspectiveControls = null; + } + this.controls = null; if (this.splatMesh) { this.splatMesh.dispose(); this.splatMesh = null; From e0fa50dfdde41187c08834f0370a330e27775ad4 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Fri, 13 Sep 2024 19:57:46 -0700 Subject: [PATCH 20/25] Cleanup --- src/Util.js | 3 +-- src/Viewer.js | 24 ++++++++--------- src/loaders/InternalLoadType.js | 5 ++++ src/loaders/ksplat/KSplatLoader.js | 8 +++--- src/loaders/ply/PlyLoader.js | 37 +++++++++++++++++++-------- src/loaders/splat/SplatLoader.js | 41 +++++++++++++++++++++--------- src/worker/SortWorker.js | 3 ++- 7 files changed, 79 insertions(+), 42 deletions(-) create mode 100644 src/loaders/InternalLoadType.js diff --git a/src/Util.js b/src/Util.js index 24ac8375..4fbb40b8 100644 --- a/src/Util.js +++ b/src/Util.js @@ -107,8 +107,7 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { chunks.push(chunk); } if (onProgress) { - const cancelSaveChunks = onProgress(percent, percentLabel, chunk, fileSize); - if (cancelSaveChunks) saveChunks = false; + onProgress(percent, percentLabel, chunk, fileSize); } } catch (error) { reject(error); diff --git a/src/Viewer.js b/src/Viewer.js index b9a76b0e..673ecbde 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -169,7 +169,7 @@ export class Viewer { if (options.optimizeSplatData === undefined || options.optimizeSplatData === null) { options.optimizeSplatData = true; } - this.optimizeSplatData = options.optimizeSplatData || false; + this.optimizeSplatData = options.optimizeSplatData; // When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and is used to // populate the data textures will be freed. This will reduces memory usage, but if that data needs to be modified @@ -202,7 +202,8 @@ export class Viewer { // Set the range for the depth map for the counting sort used to sort the splats this.splatSortDistanceMapPrecision = options.splatSortDistanceMapPrecision || Constants.DefaultSplatSortDistanceMapPrecision; - this.splatSortDistanceMapPrecision = clamp(this.splatSortDistanceMapPrecision, 10, 24); + const maxPrecision = this.integerBasedSort ? 20 : 24; + this.splatSortDistanceMapPrecision = clamp(this.splatSortDistanceMapPrecision, 10, maxPrecision); this.onSplatMeshChangedCallback = null; this.createSplatMesh(); @@ -740,7 +741,7 @@ export class Viewer { } const format = (options.format !== undefined && options.format !== null) ? options.format : sceneFormatFromPath(path); - const progressiveLoad = Viewer.isProgressivelyLoadable(format) && options.progressiveLoad && !this.optimizeSplatData; + const progressiveLoad = Viewer.isProgressivelyLoadable(format) && options.progressiveLoad; const showLoadingUI = (options.showLoadingUI !== undefined && options.showLoadingUI !== null) ? options.showLoadingUI : true; let loadingUITaskId = null; @@ -915,7 +916,7 @@ export class Viewer { const splatSceneDownloadPromise = this.downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold, onDownloadProgress, true, onProgressiveLoadSectionProgress, format); - let progressiveLoadFirstSectionBuildPromise = abortablePromiseWithExtractedComponents(splatSceneDownloadPromise.abortHandler); + const progressiveLoadFirstSectionBuildPromise = abortablePromiseWithExtractedComponents(splatSceneDownloadPromise.abortHandler); const splatSceneDownloadAndBuildPromise = abortablePromiseWithExtractedComponents(); this.addSplatSceneDownloadPromise(splatSceneDownloadPromise); @@ -1044,17 +1045,19 @@ export class Viewer { */ downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold = 1, onProgress = undefined, progressiveBuild = false, onSectionBuilt = undefined, format) { + + const optimizeSplatData = progressiveBuild ? false : this.optimizeSplatData; try { if (format === SceneFormat.Splat) { return SplatLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, splatAlphaRemovalThreshold, - this.inMemoryCompressionLevel, this.optimizeSplatData); + this.inMemoryCompressionLevel, optimizeSplatData); } else if (format === SceneFormat.KSplat) { return KSplatLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt); } else if (format === SceneFormat.Ply) { return PlyLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, splatAlphaRemovalThreshold, this.inMemoryCompressionLevel, - this.optimizeSplatData, this.sphericalHarmonicsDegree); + optimizeSplatData, this.sphericalHarmonicsDegree); } } catch (e) { if (e instanceof DirectLoadError) { @@ -1545,8 +1548,7 @@ export class Viewer { const changeEpsilon = 0.0001; return function() { - if (!this.initialized || !this.splatRenderReady) return false; - if (this.isDisposingOrDisposed()) return false; + if (!this.initialized || !this.splatRenderReady || this.isDisposingOrDisposed()) return false; let shouldRender = false; let cameraChanged = false; @@ -1579,8 +1581,7 @@ export class Viewer { render = function() { return function() { - if (!this.initialized || !this.splatRenderReady) return; - if (this.isDisposingOrDisposed()) return; + if (!this.initialized || !this.splatRenderReady || this.isDisposingOrDisposed()) return; const hasRenderables = (threeScene) => { for (let child of threeScene.children) { @@ -1606,8 +1607,7 @@ export class Viewer { update(renderer, camera) { if (this.dropInMode) this.updateForDropInMode(renderer, camera); - if (!this.initialized || !this.splatRenderReady) return; - if (this.isDisposingOrDisposed()) return; + if (!this.initialized || !this.splatRenderReady || this.isDisposingOrDisposed()) return; if (this.controls) { this.controls.update(); diff --git a/src/loaders/InternalLoadType.js b/src/loaders/InternalLoadType.js new file mode 100644 index 00000000..f3e37e18 --- /dev/null +++ b/src/loaders/InternalLoadType.js @@ -0,0 +1,5 @@ +export const InternalLoadType = { + DirectToSplatBuffer: 0, + DirectToSplatArray: 1, + DownloadBeforeProcessing: 2 +}; diff --git a/src/loaders/ksplat/KSplatLoader.js b/src/loaders/ksplat/KSplatLoader.js index f09eee14..95973cab 100644 --- a/src/loaders/ksplat/KSplatLoader.js +++ b/src/loaders/ksplat/KSplatLoader.js @@ -19,7 +19,7 @@ export class KSplatLoader { } }; - static loadFromURL(fileName, externalOnProgress, directLoad, onSectionBuilt) { + static loadFromURL(fileName, externalOnProgress, loadDirectoToSplatBuffer, onSectionBuilt) { let directLoadBuffer; let directLoadSplatBuffer; @@ -187,7 +187,7 @@ export class KSplatLoader { } numBytesLoaded += chunk.byteLength; } - if (directLoad) { + if (loadDirectoToSplatBuffer) { checkAndLoadHeader(); checkAndLoadSectionHeaders(); checkAndLoadSections(); @@ -196,9 +196,9 @@ export class KSplatLoader { } }; - return fetchWithProgress(fileName, localOnProgress, !directLoad).then((fullBuffer) => { + return fetchWithProgress(fileName, localOnProgress, !loadDirectoToSplatBuffer).then((fullBuffer) => { if (externalOnProgress) externalOnProgress(0, '0%', LoaderStatus.Processing); - const loadPromise = directLoad ? directLoadPromise.promise : KSplatLoader.loadFromFileData(fullBuffer); + const loadPromise = loadDirectoToSplatBuffer ? directLoadPromise.promise : KSplatLoader.loadFromFileData(fullBuffer); return loadPromise.then((splatBuffer) => { if (externalOnProgress) externalOnProgress(100, '100%', LoaderStatus.Done); return splatBuffer; diff --git a/src/loaders/ply/PlyLoader.js b/src/loaders/ply/PlyLoader.js index 54185a21..a89c7ec8 100644 --- a/src/loaders/ply/PlyLoader.js +++ b/src/loaders/ply/PlyLoader.js @@ -11,6 +11,7 @@ import { LoaderStatus } from '../LoaderStatus.js'; import { DirectLoadError } from '../DirectLoadError.js'; import { Constants } from '../../Constants.js'; import { UncompressedSplatArray } from '../UncompressedSplatArray.js'; +import { InternalLoadType } from '../InternalLoadType.js'; function storeChunksInBuffer(chunks, buffer) { let inBytes = 0; @@ -42,11 +43,11 @@ function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, export class PlyLoader { - static loadFromURL(fileName, onProgress, directLoad, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, + static loadFromURL(fileName, onProgress, loadDirectoToSplatBuffer, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, optimizeSplatData = true, outSphericalHarmonicsDegree = 0, sectionSize, sceneCenter, blockSize, bucketSize) { - const directLoadOriginalValue = directLoad; - if (optimizeSplatData) directLoad = false; + let internalLoadType = loadDirectoToSplatBuffer ? InternalLoadType.DirectToSplatBuffer : InternalLoadType.DirectToSplatArray; + if (optimizeSplatData) internalLoadType = InternalLoadType.DirectToSplatArray; const directLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; const splatDataOffsetBytes = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes; @@ -90,6 +91,13 @@ export class PlyLoader { numBytesDownloaded += chunkData.byteLength; } + if (internalLoadType === InternalLoadType.DownloadBeforeProcessing) { + if (loadComplete) { + loadPromise.resolve(chunks); + } + return; + } + if (!headerLoaded) { headerText += textDecoder.decode(chunkData); if (PlyParserUtils.checkTextForEndHeader(headerText)) { @@ -104,10 +112,11 @@ export class PlyLoader { maxSplatCount = header.vertexElement.count; compressed = true; } else { - if (directLoadOriginalValue) { + if (loadDirectoToSplatBuffer) { throw new DirectLoadError('PlyLoader.loadFromURL() -> Selected Ply format cannot be directly loaded.'); } else { - directLoad = false; + internalLoadType = InternalLoadType.DownloadBeforeProcessing; + return; } } outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); @@ -115,7 +124,7 @@ export class PlyLoader { const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; const splatBufferSizeBytes = splatDataOffsetBytes + shDescriptor.BytesPerSplat * maxSplatCount; - if (directLoad) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { directLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); SplatBuffer.writeHeaderToBuffer({ versionMajor: SplatBuffer.CurrentMajorVersion, @@ -166,7 +175,7 @@ export class PlyLoader { const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; const outOffset = splatCount * shDescriptor.BytesPerSplat + splatDataOffsetBytes; - if (directLoad) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { if (compressed) { PlayCanvasCompressedPlyParser.parseToUncompressedSplatBufferSection(header.chunkElement, header.vertexElement, 0, @@ -194,7 +203,7 @@ export class PlyLoader { splatCount = newSplatCount; - if (directLoad) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { if (!directLoadSplatBuffer) { SplatBuffer.writeSectionHeaderToBuffer({ maxSplatCount: maxSplatCount, @@ -236,7 +245,7 @@ export class PlyLoader { } if (loadComplete) { - if (directLoad) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { loadPromise.resolve(directLoadSplatBuffer); } else { loadPromise.resolve(standardLoadUncompressedSplatArray); @@ -252,7 +261,13 @@ export class PlyLoader { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); return loadPromise.promise.then((splatData) => { if (onProgress) onProgress(100, '100%', LoaderStatus.Done); - if (directLoad) { + if (internalLoadType === InternalLoadType.DownloadBeforeProcessing) { + const chunkDatas = chunks.map((chunk) => chunk.data); + return new Blob(chunkDatas).arrayBuffer().then((plyFileData) => { + return PlyLoader.loadFromFileData(plyFileData, minimumAlpha, compressionLevel, optimizeSplatData, + outSphericalHarmonicsDegree, sectionSize, sceneCenter, blockSize, bucketSize); + }); + } else if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { return splatData; } else { return delayedExecute(() => { @@ -264,7 +279,7 @@ export class PlyLoader { }); } - static loadFromFileData(plyFileData, minimumAlpha, compressionLevel, outSphericalHarmonicsDegree = 0, + static loadFromFileData(plyFileData, minimumAlpha, compressionLevel, optimizeSplatData, outSphericalHarmonicsDegree = 0, sectionSize, sceneCenter, blockSize, bucketSize) { return delayedExecute(() => { return PlyParser.parseToUncompressedSplatArray(plyFileData, outSphericalHarmonicsDegree); diff --git a/src/loaders/splat/SplatLoader.js b/src/loaders/splat/SplatLoader.js index 31313ba7..59a535da 100644 --- a/src/loaders/splat/SplatLoader.js +++ b/src/loaders/splat/SplatLoader.js @@ -7,6 +7,7 @@ import { UncompressedSplatArray } from '../UncompressedSplatArray.js'; import { LoaderStatus } from '../LoaderStatus.js'; import { DirectLoadError } from '../DirectLoadError.js'; import { Constants } from '../../Constants.js'; +import { InternalLoadType } from '../InternalLoadType.js'; function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, sectionSize, sceneCenter, blockSize, bucketSize) { if (optimizeSplatData) { @@ -21,11 +22,11 @@ function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, export class SplatLoader { - static loadFromURL(fileName, onProgress, directLoad, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, + static loadFromURL(fileName, onProgress, loadDirectoToSplatBuffer, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, optimizeSplatData = true, sectionSize, sceneCenter, blockSize, bucketSize) { - const directLoadOriginalValue = directLoad; - if (optimizeSplatData) directLoad = false; + let internalLoadType = loadDirectoToSplatBuffer ? InternalLoadType.DirectToSplatBuffer : InternalLoadType.DirectToSplatArray; + if (optimizeSplatData) internalLoadType = InternalLoadType.DirectToSplatArray; const splatDataOffsetBytes = SplatBuffer.HeaderSizeBytes + SplatBuffer.SectionHeaderSizeBytes; const directLoadSectionSizeBytes = Constants.ProgressiveLoadSectionSize; @@ -47,11 +48,24 @@ export class SplatLoader { const localOnProgress = (percent, percentStr, chunk, fileSize) => { const loadComplete = percent >= 100; + + if (chunk) { + chunks.push(chunk); + } + + if (internalLoadType === InternalLoadType.DownloadBeforeProcessing) { + if (loadComplete) { + loadPromise.resolve(chunks); + } + return; + } + if (!fileSize) { - if (directLoadOriginalValue) { + if (loadDirectoToSplatBuffer) { throw new DirectLoadError('Cannon directly load .splat because no file size info is available.'); } else { - directLoad = false; + internalLoadType = InternalLoadType.DownloadBeforeProcessing; + return; } } @@ -61,7 +75,7 @@ export class SplatLoader { const bytesPerSplat = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[0].BytesPerSplat; const splatBufferSizeBytes = splatDataOffsetBytes + bytesPerSplat * maxSplatCount; - if (directLoad) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { directLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); SplatBuffer.writeHeaderToBuffer({ versionMajor: SplatBuffer.CurrentMajorVersion, @@ -79,7 +93,6 @@ export class SplatLoader { } if (chunk) { - chunks.push(chunk); new Uint8Array(directLoadBufferIn, numBytesLoaded, chunk.byteLength).set(new Uint8Array(chunk)); numBytesLoaded += chunk.byteLength; @@ -89,7 +102,7 @@ export class SplatLoader { const addedSplatCount = bytesToUpdate / SplatParser.RowSizeBytes; const newSplatCount = splatCount + addedSplatCount; - if (directLoad) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { SplatParser.parseToUncompressedSplatBufferSection(splatCount, newSplatCount - 1, directLoadBufferIn, 0, directLoadBufferOut, splatDataOffsetBytes); } else { @@ -99,7 +112,7 @@ export class SplatLoader { splatCount = newSplatCount; - if (directLoad) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { if (!directLoadSplatBuffer) { SplatBuffer.writeSectionHeaderToBuffer({ maxSplatCount: maxSplatCount, @@ -125,7 +138,7 @@ export class SplatLoader { } if (loadComplete) { - if (directLoad) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { loadPromise.resolve(directLoadSplatBuffer); } else { loadPromise.resolve(standardLoadUncompressedSplatArray); @@ -133,7 +146,6 @@ export class SplatLoader { } if (onProgress) onProgress(percent, percentStr, LoaderStatus.Downloading); - return directLoad; }; if (onProgress) onProgress(0, '0%', LoaderStatus.Downloading); @@ -141,7 +153,12 @@ export class SplatLoader { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); return loadPromise.promise.then((splatData) => { if (onProgress) onProgress(100, '100%', LoaderStatus.Done); - if (directLoad) { + if (internalLoadType === InternalLoadType.DownloadBeforeProcessing) { + return new Blob(chunks).arrayBuffer().then((splatData) => { + return SplatLoader.loadFromFileData(splatData, minimumAlpha, compressionLevel, optimizeSplatData, + sectionSize, sceneCenter, blockSize, bucketSize); + }); + } else if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { return splatData; } else { return delayedExecute(() => { diff --git a/src/worker/SortWorker.js b/src/worker/SortWorker.js index 2ff4704c..7b4f0769 100644 --- a/src/worker/SortWorker.js +++ b/src/worker/SortWorker.js @@ -135,7 +135,8 @@ function sortWorker(self) { (splatCount * Constants.BytesPerInt) : (splatCount * Constants.BytesPerFloat); const memoryRequiredForMappedDistances = splatCount * Constants.BytesPerInt; const memoryRequiredForSortedIndexes = splatCount * Constants.BytesPerInt; - const memoryRequiredForIntermediateSortBuffers = distanceMapRange * Constants.BytesPerInt * 2; + const memoryRequiredForIntermediateSortBuffers = integerBasedSort ? (distanceMapRange * Constants.BytesPerInt * 2) : + (distanceMapRange * Constants.BytesPerFloat * 2); const memoryRequiredforTransformIndexes = dynamicMode ? (splatCount * Constants.BytesPerInt) : 0; const memoryRequiredforTransforms = dynamicMode ? (Constants.MaxScenes * matrixSize) : 0; const extraMemory = Constants.MemoryPageSize * 32; From bfd85906ed5fe6c9fd123fd50531e8d2c6569425 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sat, 14 Sep 2024 17:10:54 -0700 Subject: [PATCH 21/25] Optimize splat data by default when exporting to .ksplat --- demo/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demo/index.html b/demo/index.html index 29916fb9..2810f68d 100644 --- a/demo/index.html +++ b/demo/index.html @@ -294,10 +294,10 @@ function fileBufferToSplatBuffer(fileBufferData, format, alphaRemovalThreshold, compressionLevel, sectionSize, sceneCenter, blockSize, bucketSize, outSphericalHarmonicsDegree = 0) { if (format === GaussianSplats3D.SceneFormat.Ply) { - return GaussianSplats3D.PlyLoader.loadFromFileData(fileBufferData.data, alphaRemovalThreshold, compressionLevel, outSphericalHarmonicsDegree, sectionSize, sceneCenter, blockSize, bucketSize); + return GaussianSplats3D.PlyLoader.loadFromFileData(fileBufferData.data, alphaRemovalThreshold, compressionLevel, true, outSphericalHarmonicsDegree, sectionSize, sceneCenter, blockSize, bucketSize); } else { if (format === GaussianSplats3D.SceneFormat.Splat) { - return GaussianSplats3D.SplatLoader.loadFromFileData(fileBufferData.data, alphaRemovalThreshold, compressionLevel, sectionSize, sceneCenter, blockSize, bucketSize); + return GaussianSplats3D.SplatLoader.loadFromFileData(fileBufferData.data, alphaRemovalThreshold, compressionLevel, true, sectionSize, sceneCenter, blockSize, bucketSize); } else { return GaussianSplats3D.KSplatLoader.loadFromFileData(fileBufferData.data); } From 1767c697babe8dadc0e3ad8070cf7b2e008836ac Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Sun, 15 Sep 2024 07:54:02 -0700 Subject: [PATCH 22/25] Fix sort & render of more splats than have been uploaded to sort worker --- src/Viewer.js | 107 +++++++------- src/loaders/ply/PlyLoader.js | 269 +++++++++++++++++------------------ src/splatmesh/SplatMesh.js | 18 +-- src/worker/SortWorker.js | 11 +- 4 files changed, 203 insertions(+), 202 deletions(-) diff --git a/src/Viewer.js b/src/Viewer.js index 673ecbde..b0c8f89c 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -224,10 +224,13 @@ export class Viewer { this.sortWorker = null; this.sortRunning = false; this.splatRenderCount = 0; + this.splatSortCount = 0; + this.lastSplatSortCount = 0; this.sortWorkerIndexesToSort = null; this.sortWorkerSortedIndexes = null; this.sortWorkerPrecomputedDistances = null; this.sortWorkerTransforms = null; + this.preSortPosts = []; this.runAfterNextSort = []; this.selfDrivenModeRunning = false; @@ -808,7 +811,7 @@ export class Viewer { }; return this.addSplatBuffers([splatBuffer], [addSplatBufferOptions], finalBuild, firstBuild && showLoadingUI, showLoadingUI, - progressiveLoad, progressiveLoad).then(() => { + progressiveLoad).then(() => { if (!progressiveLoad && options.onProgress) options.onProgress(100, '100%', LoaderStatus.Processing); splatBuffersAddedUIUpdate(firstBuild, finalBuild); }); @@ -1005,7 +1008,7 @@ export class Viewer { .then((splatBuffers) => { if (showLoadingUI) this.loadingSpinner.removeTask(loadingUITaskId); if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); - this.addSplatBuffers(splatBuffers, sceneOptions, true, showLoadingUI, showLoadingUI, false, false).then(() => { + this.addSplatBuffers(splatBuffers, sceneOptions, true, showLoadingUI, showLoadingUI, false).then(() => { if (onProgress) onProgress(100, '100%', LoaderStatus.Processing); this.clearSplatSceneDownloadAndBuildPromise(); resolve(); @@ -1081,12 +1084,10 @@ export class Viewer { addSplatBuffers = function() { return function(splatBuffers, splatBufferOptions = [], finalBuild = true, showLoadingUI = true, - showLoadingUIForSplatTreeBuild = true, replaceExisting = false, - enableRenderBeforeFirstSort = false, preserveVisibleRegion = true) { + showLoadingUIForSplatTreeBuild = true, replaceExisting = false, preserveVisibleRegion = true) { if (this.isDisposingOrDisposed()) return Promise.resolve(); - this.splatRenderReady = false; let splatProcessingTaskId = null; const removeSplatProcessingTask = () => { @@ -1096,45 +1097,6 @@ export class Viewer { } }; - const finish = (buildResults, resolver) => { - if (this.isDisposingOrDisposed()) return; - - // If we aren't calculating the splat distances from the center on the GPU, the sorting worker needs splat centers and - // transform indexes so that it can calculate those distance values. - if (!this.gpuAcceleratedSort && this.sortWorker) { - this.sortWorker.postMessage({ - 'centers': buildResults.centers.buffer, - 'sceneIndexes': buildResults.sceneIndexes.buffer, - 'range': { - 'from': buildResults.from, - 'to': buildResults.to, - 'count': buildResults.count - } - }); - } - - this.runSplatSort(true).then((sortRunning) => { - if (!this.sortWorker || !sortRunning) { - this.splatRenderReady = true; - removeSplatProcessingTask(); - resolver(); - } else { - if (enableRenderBeforeFirstSort) { - this.splatRenderReady = true; - } else { - this.runAfterNextSort.push(() => { - this.splatRenderReady = true; - }); - } - this.runAfterNextSort.push(() => { - removeSplatProcessingTask(); - resolver(); - }); - } - }); - - }; - return new Promise((resolve) => { if (showLoadingUI) { splatProcessingTaskId = this.loadingSpinner.addTask('Processing splats...'); @@ -1146,12 +1108,38 @@ export class Viewer { const buildResults = this.addSplatBuffersToMesh(splatBuffers, splatBufferOptions, finalBuild, showLoadingUIForSplatTreeBuild, replaceExisting, preserveVisibleRegion); + // If we aren't calculating the splat distances from the center on the GPU, the sorting worker needs + // splat centers and transform indexes so that it can calculate those distance values. + if (!this.gpuAcceleratedSort) { + this.preSortPosts.push({ + 'centers': buildResults.centers.buffer, + 'sceneIndexes': buildResults.sceneIndexes.buffer, + 'range': { + 'from': buildResults.from, + 'to': buildResults.to, + 'count': buildResults.count + } + }); + } + const maxSplatCount = this.splatMesh.getMaxSplatCount(); if (this.sortWorker && this.sortWorker.maxSplatCount !== maxSplatCount) this.disposeSortWorker(); const sortWorkerSetupPromise = (!this.sortWorker && maxSplatCount > 0) ? this.setupSortWorker(this.splatMesh) : Promise.resolve(); sortWorkerSetupPromise.then(() => { - finish(buildResults, resolve); + if (this.isDisposingOrDisposed()) return; + if (this.sortWorker) { + if (!this.sortRunning) { + this.runSplatSort(true, true); + } else if (finalBuild) { + this.sortPromise.then(() => { + this.runSplatSort(true, true); + }); + } + } + this.splatRenderReady = true; + removeSplatProcessingTask(); + resolve(); }); } }, true); @@ -1243,6 +1231,9 @@ export class Viewer { const sortedIndexes = new Uint32Array(e.data.sortedIndexes.buffer, 0, e.data.splatRenderCount); this.splatMesh.updateRenderIndexes(sortedIndexes, e.data.splatRenderCount); } + + this.lastSplatSortCount = this.splatSortCount; + this.lastSortTime = e.data.sortTime; this.sortPromiseResolver(); this.sortPromiseResolver = null; @@ -1837,7 +1828,7 @@ export class Viewer { } ]; - return function(force = false) { + return function(force = false, forceSortAll = false) { if (!this.initialized) return Promise.resolve(false); if (this.sortRunning) return Promise.resolve(true); if (this.splatMesh.getSplatCount() <= 0) { @@ -1863,7 +1854,8 @@ export class Viewer { } this.sortRunning = true; - const { splatRenderCount, shouldSortAll } = this.gatherSceneNodesForSort(); + let { splatRenderCount, shouldSortAll } = this.gatherSceneNodesForSort(); + shouldSortAll = shouldSortAll || forceSortAll; this.splatRenderCount = splatRenderCount; mvpMatrix.copy(this.camera.matrixWorld).invert(); @@ -1871,17 +1863,17 @@ export class Viewer { mvpMatrix.premultiply(mvpCamera.projectionMatrix); mvpMatrix.multiply(this.splatMesh.matrixWorld); - let gpuAcceleratedSortPromise = Promise.resolve(); + let gpuAcceleratedSortPromise = Promise.resolve(true); if (this.gpuAcceleratedSort && (queuedSorts.length <= 1 || queuedSorts.length % 2 === 0)) { gpuAcceleratedSortPromise = this.splatMesh.computeDistancesOnGPU(mvpMatrix, this.sortWorkerPrecomputedDistances); } gpuAcceleratedSortPromise.then(() => { - if (this.splatMesh.dynamicMode || shouldSortAll) { - queuedSorts.push(this.splatRenderCount); - } else { - if (queuedSorts.length === 0) { - for (let partialSort of partialSorts) { + if (queuedSorts.length === 0) { + if (this.splatMesh.dynamicMode || shouldSortAll) { + queuedSorts.push(this.splatRenderCount); + } else { + for (let partialSort of partialSorts) { if (angleDiff < partialSort.angleThreshold) { for (let sortFraction of partialSort.sortFractions) { queuedSorts.push(Math.floor(this.splatRenderCount * sortFraction)); @@ -1893,6 +1885,7 @@ export class Viewer { } } let sortCount = Math.min(queuedSorts.shift(), this.splatRenderCount); + this.splatSortCount = sortCount; cameraPositionArray[0] = this.camera.position.x; cameraPositionArray[1] = this.camera.position.y; @@ -1920,6 +1913,12 @@ export class Viewer { this.sortPromiseResolver = resolve; }); + if (this.preSortPosts.length > 0) { + this.preSortPosts.forEach((message) => { + this.sortWorker.postMessage(message); + }); + this.preSortPosts = []; + } this.sortWorker.postMessage({ 'sort': sortMessage }); @@ -1928,6 +1927,8 @@ export class Viewer { lastSortViewPos.copy(this.camera.position); lastSortViewDir.copy(sortViewDir); } + + return true; }); return gpuAcceleratedSortPromise; diff --git a/src/loaders/ply/PlyLoader.js b/src/loaders/ply/PlyLoader.js index a89c7ec8..4dabc9bf 100644 --- a/src/loaders/ply/PlyLoader.js +++ b/src/loaders/ply/PlyLoader.js @@ -95,160 +95,159 @@ export class PlyLoader { if (loadComplete) { loadPromise.resolve(chunks); } - return; - } + } else { + if (!headerLoaded) { + headerText += textDecoder.decode(chunkData); + if (PlyParserUtils.checkTextForEndHeader(headerText)) { + const plyFormat = PlyParserUtils.determineHeaderFormatFromHeaderText(headerText); + if (plyFormat === PlyFormat.INRIAV1) { + header = inriaV1PlyParser.decodeHeaderText(headerText); + maxSplatCount = header.splatCount; + readyToLoadSplatData = true; + compressed = false; + } else if (plyFormat === PlyFormat.PlayCanvasCompressed) { + header = PlayCanvasCompressedPlyParser.decodeHeaderText(headerText); + maxSplatCount = header.vertexElement.count; + compressed = true; + } else { + if (loadDirectoToSplatBuffer) { + throw new DirectLoadError('PlyLoader.loadFromURL() -> Selected Ply format cannot be directly loaded.'); + } else { + internalLoadType = InternalLoadType.DownloadBeforeProcessing; + return; + } + } + outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); - if (!headerLoaded) { - headerText += textDecoder.decode(chunkData); - if (PlyParserUtils.checkTextForEndHeader(headerText)) { - const plyFormat = PlyParserUtils.determineHeaderFormatFromHeaderText(headerText); - if (plyFormat === PlyFormat.INRIAV1) { - header = inriaV1PlyParser.decodeHeaderText(headerText); - maxSplatCount = header.splatCount; - readyToLoadSplatData = true; - compressed = false; - } else if (plyFormat === PlyFormat.PlayCanvasCompressed) { - header = PlayCanvasCompressedPlyParser.decodeHeaderText(headerText); - maxSplatCount = header.vertexElement.count; - compressed = true; - } else { - if (loadDirectoToSplatBuffer) { - throw new DirectLoadError('PlyLoader.loadFromURL() -> Selected Ply format cannot be directly loaded.'); + const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; + const splatBufferSizeBytes = splatDataOffsetBytes + shDescriptor.BytesPerSplat * maxSplatCount; + + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + directLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); + SplatBuffer.writeHeaderToBuffer({ + versionMajor: SplatBuffer.CurrentMajorVersion, + versionMinor: SplatBuffer.CurrentMinorVersion, + maxSectionCount: sectionCount, + sectionCount: sectionCount, + maxSplatCount: maxSplatCount, + splatCount: splatCount, + compressionLevel: 0, + sceneCenter: new THREE.Vector3() + }, directLoadBufferOut); } else { - internalLoadType = InternalLoadType.DownloadBeforeProcessing; - return; + standardLoadUncompressedSplatArray = new UncompressedSplatArray(outSphericalHarmonicsDegree); } + + numBytesStreamed = header.headerSizeBytes; + numBytesParsed = header.headerSizeBytes; + headerLoaded = true; } - outSphericalHarmonicsDegree = Math.min(outSphericalHarmonicsDegree, header.sphericalHarmonicsDegree); - - const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; - const splatBufferSizeBytes = splatDataOffsetBytes + shDescriptor.BytesPerSplat * maxSplatCount; - - if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { - directLoadBufferOut = new ArrayBuffer(splatBufferSizeBytes); - SplatBuffer.writeHeaderToBuffer({ - versionMajor: SplatBuffer.CurrentMajorVersion, - versionMinor: SplatBuffer.CurrentMinorVersion, - maxSectionCount: sectionCount, - sectionCount: sectionCount, - maxSplatCount: maxSplatCount, - splatCount: splatCount, - compressionLevel: 0, - sceneCenter: new THREE.Vector3() - }, directLoadBufferOut); - } else { - standardLoadUncompressedSplatArray = new UncompressedSplatArray(outSphericalHarmonicsDegree); + } else if (compressed && !readyToLoadSplatData) { + const sizeRequiredForHeaderAndChunks = header.headerSizeBytes + header.chunkElement.storageSizeBytes; + compressedPlyHeaderChunksBuffer = storeChunksInBuffer(chunks, compressedPlyHeaderChunksBuffer); + if (compressedPlyHeaderChunksBuffer.byteLength >= sizeRequiredForHeaderAndChunks) { + PlayCanvasCompressedPlyParser.readElementData(header.chunkElement, compressedPlyHeaderChunksBuffer, + header.headerSizeBytes); + numBytesStreamed = sizeRequiredForHeaderAndChunks; + numBytesParsed = sizeRequiredForHeaderAndChunks; + readyToLoadSplatData = true; } - - numBytesStreamed = header.headerSizeBytes; - numBytesParsed = header.headerSizeBytes; - headerLoaded = true; - } - } else if (compressed && !readyToLoadSplatData) { - const sizeRequiredForHeaderAndChunks = header.headerSizeBytes + header.chunkElement.storageSizeBytes; - compressedPlyHeaderChunksBuffer = storeChunksInBuffer(chunks, compressedPlyHeaderChunksBuffer); - if (compressedPlyHeaderChunksBuffer.byteLength >= sizeRequiredForHeaderAndChunks) { - PlayCanvasCompressedPlyParser.readElementData(header.chunkElement, compressedPlyHeaderChunksBuffer, - header.headerSizeBytes); - numBytesStreamed = sizeRequiredForHeaderAndChunks; - numBytesParsed = sizeRequiredForHeaderAndChunks; - readyToLoadSplatData = true; } - } - - if (headerLoaded && readyToLoadSplatData) { - - if (chunks.length > 0) { - - directLoadBufferIn = storeChunksInBuffer(chunks, directLoadBufferIn); - - const bytesLoadedSinceLastStreamedSection = numBytesDownloaded - numBytesStreamed; - if (bytesLoadedSinceLastStreamedSection > directLoadSectionSizeBytes || loadComplete) { - const numBytesToProcess = numBytesDownloaded - numBytesParsed; - const addedSplatCount = Math.floor(numBytesToProcess / header.bytesPerSplat); - const numBytesToParse = addedSplatCount * header.bytesPerSplat; - const numBytesLeftOver = numBytesToProcess - numBytesToParse; - const newSplatCount = splatCount + addedSplatCount; - const parsedDataViewOffset = numBytesParsed - chunks[0].startBytes; - const dataToParse = new DataView(directLoadBufferIn, parsedDataViewOffset, numBytesToParse); - - const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; - const outOffset = splatCount * shDescriptor.BytesPerSplat + splatDataOffsetBytes; - if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { - if (compressed) { - PlayCanvasCompressedPlyParser.parseToUncompressedSplatBufferSection(header.chunkElement, + if (headerLoaded && readyToLoadSplatData) { + + if (chunks.length > 0) { + + directLoadBufferIn = storeChunksInBuffer(chunks, directLoadBufferIn); + + const bytesLoadedSinceLastStreamedSection = numBytesDownloaded - numBytesStreamed; + if (bytesLoadedSinceLastStreamedSection > directLoadSectionSizeBytes || loadComplete) { + const numBytesToProcess = numBytesDownloaded - numBytesParsed; + const addedSplatCount = Math.floor(numBytesToProcess / header.bytesPerSplat); + const numBytesToParse = addedSplatCount * header.bytesPerSplat; + const numBytesLeftOver = numBytesToProcess - numBytesToParse; + const newSplatCount = splatCount + addedSplatCount; + const parsedDataViewOffset = numBytesParsed - chunks[0].startBytes; + const dataToParse = new DataView(directLoadBufferIn, parsedDataViewOffset, numBytesToParse); + + const shDescriptor = SplatBuffer.CompressionLevels[0].SphericalHarmonicsDegrees[outSphericalHarmonicsDegree]; + const outOffset = splatCount * shDescriptor.BytesPerSplat + splatDataOffsetBytes; + + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + if (compressed) { + PlayCanvasCompressedPlyParser.parseToUncompressedSplatBufferSection(header.chunkElement, + header.vertexElement, 0, + addedSplatCount - 1, splatCount, + dataToParse, 0, + directLoadBufferOut, outOffset); + } else { + inriaV1PlyParser.parseToUncompressedSplatBufferSection(header, 0, addedSplatCount - 1, dataToParse, + 0, directLoadBufferOut, outOffset, + outSphericalHarmonicsDegree); + } + } else { + if (compressed) { + PlayCanvasCompressedPlyParser.parseToUncompressedSplatArraySection(header.chunkElement, header.vertexElement, 0, addedSplatCount - 1, splatCount, dataToParse, 0, - directLoadBufferOut, outOffset); - } else { - inriaV1PlyParser.parseToUncompressedSplatBufferSection(header, 0, addedSplatCount - 1, dataToParse, - 0, directLoadBufferOut, outOffset, - outSphericalHarmonicsDegree); - } - } else { - if (compressed) { - PlayCanvasCompressedPlyParser.parseToUncompressedSplatArraySection(header.chunkElement, - header.vertexElement, 0, - addedSplatCount - 1, splatCount, - dataToParse, 0, - standardLoadUncompressedSplatArray); - } else { - inriaV1PlyParser.parseToUncompressedSplatArraySection(header, 0, addedSplatCount - 1, dataToParse, - 0, standardLoadUncompressedSplatArray, - outSphericalHarmonicsDegree); + standardLoadUncompressedSplatArray); + } else { + inriaV1PlyParser.parseToUncompressedSplatArraySection(header, 0, addedSplatCount - 1, dataToParse, + 0, standardLoadUncompressedSplatArray, + outSphericalHarmonicsDegree); + } } - } - splatCount = newSplatCount; - - if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { - if (!directLoadSplatBuffer) { - SplatBuffer.writeSectionHeaderToBuffer({ - maxSplatCount: maxSplatCount, - splatCount: splatCount, - bucketSize: 0, - bucketCount: 0, - bucketBlockSize: 0, - compressionScaleRange: 0, - storageSizeBytes: 0, - fullBucketCount: 0, - partiallyFilledBucketCount: 0, - sphericalHarmonicsDegree: outSphericalHarmonicsDegree - }, 0, directLoadBufferOut, SplatBuffer.HeaderSizeBytes); - directLoadSplatBuffer = new SplatBuffer(directLoadBufferOut, false); + splatCount = newSplatCount; + + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + if (!directLoadSplatBuffer) { + SplatBuffer.writeSectionHeaderToBuffer({ + maxSplatCount: maxSplatCount, + splatCount: splatCount, + bucketSize: 0, + bucketCount: 0, + bucketBlockSize: 0, + compressionScaleRange: 0, + storageSizeBytes: 0, + fullBucketCount: 0, + partiallyFilledBucketCount: 0, + sphericalHarmonicsDegree: outSphericalHarmonicsDegree + }, 0, directLoadBufferOut, SplatBuffer.HeaderSizeBytes); + directLoadSplatBuffer = new SplatBuffer(directLoadBufferOut, false); + } + directLoadSplatBuffer.updateLoadedCounts(1, splatCount); + if (onProgressiveLoadSectionProgress) { + onProgressiveLoadSectionProgress(directLoadSplatBuffer, loadComplete); + } } - directLoadSplatBuffer.updateLoadedCounts(1, splatCount); - if (onProgressiveLoadSectionProgress) { - onProgressiveLoadSectionProgress(directLoadSplatBuffer, loadComplete); - } - } - numBytesStreamed += directLoadSectionSizeBytes; - numBytesParsed += numBytesToParse; + numBytesStreamed += directLoadSectionSizeBytes; + numBytesParsed += numBytesToParse; - if (numBytesLeftOver === 0) { - chunks = []; - } else { - let keepChunks = []; - let keepSize = 0; - for (let i = chunks.length - 1; i >= 0; i--) { - const chunk = chunks[i]; - keepSize += chunk.sizeBytes; - keepChunks.unshift(chunk); - if (keepSize >= numBytesLeftOver) break; + if (numBytesLeftOver === 0) { + chunks = []; + } else { + let keepChunks = []; + let keepSize = 0; + for (let i = chunks.length - 1; i >= 0; i--) { + const chunk = chunks[i]; + keepSize += chunk.sizeBytes; + keepChunks.unshift(chunk); + if (keepSize >= numBytesLeftOver) break; + } + chunks = keepChunks; } - chunks = keepChunks; } } - } - if (loadComplete) { - if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { - loadPromise.resolve(directLoadSplatBuffer); - } else { - loadPromise.resolve(standardLoadUncompressedSplatArray); + if (loadComplete) { + if (internalLoadType === InternalLoadType.DirectToSplatBuffer) { + loadPromise.resolve(directLoadSplatBuffer); + } else { + loadPromise.resolve(standardLoadUncompressedSplatArray); + } } } } diff --git a/src/splatmesh/SplatMesh.js b/src/splatmesh/SplatMesh.js index 014709c1..6251efd1 100644 --- a/src/splatmesh/SplatMesh.js +++ b/src/splatmesh/SplatMesh.js @@ -375,14 +375,14 @@ export class SplatMesh extends THREE.Mesh { this.globalSplatIndexToSceneIndexMap = indexMaps.sceneIndexMap; } - const splatCount = this.getSplatCount(); + const splatBufferSplatCount = this.getSplatCount(true); if (this.enableDistancesComputationOnGPU) this.setupDistancesComputationTransformFeedback(); const dataUpdateResults = this.refreshGPUDataFromSplatBuffers(isUpdateBuild); for (let i = 0; i < this.scenes.length; i++) { this.lastBuildScenes[i] = this.scenes[i]; } - this.lastBuildSplatCount = splatCount; + this.lastBuildSplatCount = splatBufferSplatCount; this.lastBuildMaxSplatCount = this.getMaxSplatCount(); this.lastBuildSceneCount = this.scenes.length; @@ -582,7 +582,7 @@ export class SplatMesh extends THREE.Mesh { * @return {object} */ refreshGPUDataFromSplatBuffers(sinceLastBuildOnly) { - const splatCount = this.getSplatCount(); + const splatCount = this.getSplatCount(true); this.refreshDataTexturesFromSplatBuffers(sinceLastBuildOnly); const updateStart = sinceLastBuildOnly ? this.lastBuildSplatCount : 0; const { centers, sceneIndexes } = this.getDataForDistancesComputation(updateStart, splatCount - 1); @@ -615,7 +615,7 @@ export class SplatMesh extends THREE.Mesh { * @param {boolean} sinceLastBuildOnly Specify whether or not to only update for splats that have been added since the last build. */ refreshDataTexturesFromSplatBuffers(sinceLastBuildOnly) { - const splatCount = this.getSplatCount(); + const splatCount = this.getSplatCount(true); const fromSplat = this.lastBuildSplatCount; const toSplat = splatCount - 1; @@ -632,7 +632,7 @@ export class SplatMesh extends THREE.Mesh { setupDataTextures() { const maxSplatCount = this.getMaxSplatCount(); - const splatCount = this.getSplatCount(); + const splatCount = this.getSplatCount(true); this.disposeTextures(); @@ -1166,7 +1166,7 @@ export class SplatMesh extends THREE.Mesh { } updateVisibleRegion(sinceLastBuildOnly) { - const splatCount = this.getSplatCount(); + const splatCount = this.getSplatCount(true); const tempCenter = new THREE.Vector3(); if (!sinceLastBuildOnly) { const avgCenter = new THREE.Vector3(); @@ -1227,6 +1227,7 @@ export class SplatMesh extends THREE.Mesh { geometry.attributes.splatIndex.needsUpdate = true; if (renderSplatCount > 0 && this.firstRenderTime === -1) this.firstRenderTime = performance.now(); geometry.instanceCount = renderSplatCount; + geometry.setDrawRange(0, renderSplatCount); } /** @@ -1298,8 +1299,9 @@ export class SplatMesh extends THREE.Mesh { return this.splatDataTextures; } - getSplatCount() { - return SplatMesh.getTotalSplatCountForScenes(this.scenes); + getSplatCount(includeSinceLastBuild = false) { + if (!includeSinceLastBuild) return this.lastBuildSplatCount; + else return SplatMesh.getTotalSplatCountForScenes(this.scenes); } static getTotalSplatCountForScenes(scenes) { diff --git a/src/worker/SortWorker.js b/src/worker/SortWorker.js index 7b4f0769..077c5266 100644 --- a/src/worker/SortWorker.js +++ b/src/worker/SortWorker.js @@ -25,7 +25,7 @@ function sortWorker(self) { let countsZero; let sortedIndexesOut; let distanceMapRange; - + let uploadedSplatCount; let Constants; function sort(splatSortCount, splatRenderCount, modelViewProj, @@ -95,12 +95,10 @@ function sortWorker(self) { new Uint32Array(wasmMemory, sceneIndexesOffset + e.data.range.from * 4, e.data.range.count).set(new Uint32Array(sceneIndexes)); } - self.postMessage({ - 'centerDataSet': true, - }); + uploadedSplatCount = e.data.range.from + e.data.range.count; } else if (e.data.sort) { - const renderCount = e.data.sort.splatRenderCount || 0; - const sortCount = e.data.sort.splatSortCount || 0; + const renderCount = Math.min(e.data.sort.splatRenderCount || 0, uploadedSplatCount); + const sortCount = Math.min(e.data.sort.splatSortCount || 0, uploadedSplatCount); const usePrecomputedDistances = e.data.sort.usePrecomputedDistances; let copyIndexesToSort; @@ -122,6 +120,7 @@ function sortWorker(self) { integerBasedSort = e.data.init.integerBasedSort; dynamicMode = e.data.init.dynamicMode; distanceMapRange = e.data.init.distanceMapRange; + uploadedSplatCount = 0; const CENTERS_BYTES_PER_ENTRY = integerBasedSort ? (Constants.BytesPerInt * 4) : (Constants.BytesPerFloat * 4); From e8932183ba66817cd0bcc01169b4973603c5ee8c Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Mon, 16 Sep 2024 20:01:36 -0700 Subject: [PATCH 23/25] Cleanup --- src/Util.js | 4 ++-- src/Viewer.js | 53 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/Util.js b/src/Util.js index 4fbb40b8..870dbe6c 100644 --- a/src/Util.js +++ b/src/Util.js @@ -60,7 +60,7 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { const signal = abortController.signal; let aborted = false; const abortHandler = (reason) => { - abortController.abort(new AbortedPromiseError(reason)); + abortController.abort(reason); aborted = true; }; @@ -116,7 +116,7 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { } }) .catch((error) => { - reject(error); + reject(new AbortedPromiseError(error)); }); }, abortHandler); diff --git a/src/Viewer.js b/src/Viewer.js index b0c8f89c..548fc01b 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -230,7 +230,7 @@ export class Viewer { this.sortWorkerSortedIndexes = null; this.sortWorkerPrecomputedDistances = null; this.sortWorkerTransforms = null; - this.preSortPosts = []; + this.preSortMessages = []; this.runAfterNextSort = []; this.selfDrivenModeRunning = false; @@ -811,7 +811,7 @@ export class Viewer { }; return this.addSplatBuffers([splatBuffer], [addSplatBufferOptions], finalBuild, firstBuild && showLoadingUI, showLoadingUI, - progressiveLoad).then(() => { + progressiveLoad, progressiveLoad).then(() => { if (!progressiveLoad && options.onProgress) options.onProgress(100, '100%', LoaderStatus.Processing); splatBuffersAddedUIUpdate(firstBuild, finalBuild); }); @@ -1008,7 +1008,7 @@ export class Viewer { .then((splatBuffers) => { if (showLoadingUI) this.loadingSpinner.removeTask(loadingUITaskId); if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); - this.addSplatBuffers(splatBuffers, sceneOptions, true, showLoadingUI, showLoadingUI, false).then(() => { + this.addSplatBuffers(splatBuffers, sceneOptions, true, showLoadingUI, showLoadingUI, false, false).then(() => { if (onProgress) onProgress(100, '100%', LoaderStatus.Processing); this.clearSplatSceneDownloadAndBuildPromise(); resolve(); @@ -1084,12 +1084,12 @@ export class Viewer { addSplatBuffers = function() { return function(splatBuffers, splatBufferOptions = [], finalBuild = true, showLoadingUI = true, - showLoadingUIForSplatTreeBuild = true, replaceExisting = false, preserveVisibleRegion = true) { + showLoadingUIForSplatTreeBuild = true, replaceExisting = false, + enableRenderBeforeFirstSort = false, preserveVisibleRegion = true) { if (this.isDisposingOrDisposed()) return Promise.resolve(); let splatProcessingTaskId = null; - const removeSplatProcessingTask = () => { if (splatProcessingTaskId !== null) { this.loadingSpinner.removeTask(splatProcessingTaskId); @@ -1097,6 +1097,7 @@ export class Viewer { } }; + this.splatRenderReady = false; return new Promise((resolve) => { if (showLoadingUI) { splatProcessingTaskId = this.loadingSpinner.addTask('Processing splats...'); @@ -1108,10 +1109,13 @@ export class Viewer { const buildResults = this.addSplatBuffersToMesh(splatBuffers, splatBufferOptions, finalBuild, showLoadingUIForSplatTreeBuild, replaceExisting, preserveVisibleRegion); + + const maxSplatCount = this.splatMesh.getMaxSplatCount(); + if (this.sortWorker && this.sortWorker.maxSplatCount !== maxSplatCount) this.disposeSortWorker(); // If we aren't calculating the splat distances from the center on the GPU, the sorting worker needs // splat centers and transform indexes so that it can calculate those distance values. if (!this.gpuAcceleratedSort) { - this.preSortPosts.push({ + this.preSortMessages.push({ 'centers': buildResults.centers.buffer, 'sceneIndexes': buildResults.sceneIndexes.buffer, 'range': { @@ -1121,25 +1125,29 @@ export class Viewer { } }); } - - const maxSplatCount = this.splatMesh.getMaxSplatCount(); - if (this.sortWorker && this.sortWorker.maxSplatCount !== maxSplatCount) this.disposeSortWorker(); const sortWorkerSetupPromise = (!this.sortWorker && maxSplatCount > 0) ? this.setupSortWorker(this.splatMesh) : Promise.resolve(); sortWorkerSetupPromise.then(() => { if (this.isDisposingOrDisposed()) return; - if (this.sortWorker) { - if (!this.sortRunning) { - this.runSplatSort(true, true); - } else if (finalBuild) { - this.sortPromise.then(() => { - this.runSplatSort(true, true); + this.runSplatSort(true, true).then((sortRunning) => { + if (!this.sortWorker || !sortRunning) { + this.splatRenderReady = true; + removeSplatProcessingTask(); + resolve(); + } else { + if (enableRenderBeforeFirstSort) { + this.splatRenderReady = true; + } else { + this.runAfterNextSort.push(() => { + this.splatRenderReady = true; + }); + } + this.runAfterNextSort.push(() => { + removeSplatProcessingTask(); + resolve(); }); } - } - this.splatRenderReady = true; - removeSplatProcessingTask(); - resolve(); + }); }); } }, true); @@ -1289,6 +1297,7 @@ export class Viewer { this.sortPromiseResolver(); this.sortPromiseResolver = null; } + this.preSortMessages = []; this.sortRunning = false; } @@ -1913,11 +1922,11 @@ export class Viewer { this.sortPromiseResolver = resolve; }); - if (this.preSortPosts.length > 0) { - this.preSortPosts.forEach((message) => { + if (this.preSortMessages.length > 0) { + this.preSortMessages.forEach((message) => { this.sortWorker.postMessage(message); }); - this.preSortPosts = []; + this.preSortMessages = []; } this.sortWorker.postMessage({ 'sort': sortMessage From 0d80cbaffc3448b56d64cfdbbe6396713e311a44 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Tue, 17 Sep 2024 08:18:33 -0700 Subject: [PATCH 24/25] Update README --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ed58e5ea..690e85af 100644 --- a/README.md +++ b/README.md @@ -22,14 +22,22 @@ When I started, web-based viewers were already available -- A WebGL-based viewer - WASM splat sort: Implemented in C++ using WASM SIMD instructions - Partially GPU accelerated splat sort: Uses transform feedback to pre-calculate splat distances +## Tips + +- Progressively loaded `.ply` and `.splat` files will not have certain optimizations such as cache-optimized splat ordering applied to them. For optimial performance, convert these file types to `.ksplat` or load them non-progressively. +- Converting your scenes to `.ksplat` will result in the fastest loading times since its format matches the internal format for splat data. +- Scenes with large dimensions or high splat density will cause issues with the default settings. For those scenes, you can try a couple of things: + - Set the viewer parameter `integerBasedSort` to `false` to force a slower, floating-point based splat sort. + - Experiment with a larger value for viewer parameter `splatSortDistanceMapPrecision`, to adjust the precision for the distance map in the splat sort. Larger precision values will result in reduced performance, but often can alleviate visual artifacts that arise when the precision is too low. + + ## Known issues - Splat sort runs on the CPU – would be great to figure out a GPU-based approach - Artifacts are visible when you move or rotate too fast (due to CPU-based splat sort) - Sub-optimal performance on mobile devices - Custom `.ksplat` file format still needs work, especially around compression -- The default, integer based splat sort does not work well for larger scenes. In that case a value of `false` for the `integerBasedSort` viewer parameter can force a slower, floating-point based sort -- The default precision (16-bit) for the distance map in the splat sort may not work well for larger scenes, or scenes with a dense splat arrangement. For those scenes the viewer parameter `splatSortDistanceMapPrecision` can be used to adjust that value. Larger precision values will result in reduced performance, but often can alleviate visual artifacts that arise when the precision is too low. +- Scenes with very large dimensions will probably crash (often with an `Index out of bounds` error from the splat sort). Changing `splatSortDistanceMapPrecision` or `integerBasedSort` will probably not help in those cases. ## Limitations From ec8831c40c32448864c3f2bdbb01d3c2ae09f527 Mon Sep 17 00:00:00 2001 From: Mark Kellogg Date: Tue, 17 Sep 2024 13:11:25 -0700 Subject: [PATCH 25/25] Update to version v0.4.5 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7b829a9e..decfc9e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.4", + "version": "0.4.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.4", + "version": "0.4.5", "license": "MIT", "devDependencies": { "@babel/core": "7.22.0", diff --git a/package.json b/package.json index d899299e..ff08960f 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "type": "git", "url": "https://github.com/mkkellogg/GaussianSplats3D" }, - "version": "0.4.4", + "version": "0.4.5", "description": "Three.js-based 3D Gaussian splat viewer", "module": "build/gaussian-splats-3d.module.js", "main": "build/gaussian-splats-3d.umd.cjs",