diff --git a/README.md b/README.md index dc17746a..59290a8a 100644 --- a/README.md +++ b/README.md @@ -354,7 +354,13 @@ Both of the above methods will prompt your browser to automatically start downlo The third option is to use the included nodejs script: ``` -node util/create-ksplat.js [path to .PLY or .SPLAT] [output file] [compression level = 0] [alpha removal threshold = 1] +node util/create-ksplat.js [path to .PLY or .SPLAT] [output file] [compression level = 0] [alpha removal threshold = 1] [scene center = "0,0,0"] [block size = 5.0] [bucket size = 256] [spherical harmonics level = 0] +``` + +For the nodejs script, it may be necessary to increase the heap size for larger scenes. Use the parameter `--max-old-space-size=[heap size in MB]` to do so: + +``` +node util/create-ksplat.js --max-old-space-size=8192 [... remaining arguments] ``` Currently supported values for `compressionLevel` are `0`, `1`, or `2`. `0` means no compression and `1` means compression of scale, rotation, position, and spherical harmonics coefficient values from 32-bit to 16-bit. `2` is similar to `1` except spherical harmonics coefficients are compressed to 8-bit. diff --git a/demo/dynamic_dropin.html b/demo/dynamic_dropin.html new file mode 100644 index 00000000..1724a662 --- /dev/null +++ b/demo/dynamic_dropin.html @@ -0,0 +1,173 @@ + + + + + + + + 3D Gaussian Splats - Drop-in example + + + + + + + + + + + \ No newline at end of file diff --git a/demo/dynamic_scenes.html b/demo/dynamic_scenes.html index 5a1bc155..ecaa682c 100644 --- a/demo/dynamic_scenes.html +++ b/demo/dynamic_scenes.html @@ -37,7 +37,8 @@ 'initialCameraLookAt': [1.52976, 2.27776, 1.65898], 'dynamicScene': true }); - const lp = viewer.addSplatScenes([ + + viewer.addSplatScenes([ { 'path': 'assets/data/garden/garden.ksplat', 'splatAlphaRemovalThreshold': 20, @@ -64,24 +65,40 @@ const position = new THREE.Vector3(); const scale = new THREE.Vector3(1.25, 1.25, 1.25); + // generate splat mesh parent objects + const sphereGeometry = new THREE.SphereGeometry(0.25, 8, 8); + const material = new THREE.MeshBasicMaterial({color: 0xff0000}); + const meshA = new THREE.Mesh(sphereGeometry, material); + const meshB = new THREE.Mesh(sphereGeometry, material); + + // add splat mesh parent objects to the scene + viewer.splatMesh.add(meshA); + viewer.splatMesh.add(meshB); + + // You can modify the transform components (position, quaternion, scale) of a SplatScene + // directly like any three.js object OR you can just attach them to another three.js object + // and they will be transformed accordingly. Below we are going with the latter approach. + // The splat scenes at index 1 & 2 are (by default) children of viewer.splatMesh, so we + // re-parent them to meshA and meshB respectively. + meshA.add(viewer.getSplatScene(1)); + meshB.add(viewer.getSplatScene(2)); + let startTime = performance.now() / 1000.0; requestAnimationFrame(update); function update() { requestAnimationFrame(update); const timeDelta = performance.now() / 1000.0 - startTime; for (let i = bonsaiStartIndex; i < bonsaiStartIndex + bonsaiCount; i++) { + // calculate parent mesh positions & orientations const angle = timeDelta * 0.25 + (Math.PI * 2) * (i /bonsaiCount); const height = Math.cos(timeDelta + (Math.PI * 2) * (i / bonsaiCount)) * 0.5 + 3; - rotationQuaternion.setFromAxisAngle(rotationAxis, angle); horizontalOffsetVector.set(3, 0, 0).applyQuaternion(rotationQuaternion); - position.copy(rotationAxis).multiplyScalar(height).add(horizontalOffsetVector).add(orbitCenter); - quaternion.copy(baseQuaternion).premultiply(rotationQuaternion); - - const splatScene = viewer.getSplatScene(i); - splatScene.position.copy(position); - splatScene.quaternion.copy(quaternion); - splatScene.scale.copy(scale); + // apply mesh position, orientation and scale + const mesh = (i % 2 === 0) ? meshA : meshB; + mesh.position.copy(rotationAxis).multiplyScalar(height).add(horizontalOffsetVector).add(orbitCenter); + mesh.quaternion.copy(baseQuaternion).premultiply(rotationQuaternion); + mesh.scale.copy(scale); } } diff --git a/package-lock.json b/package-lock.json index 7f8c763d..7b829a9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.3", + "version": "0.4.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.3", + "version": "0.4.4", "license": "MIT", "devDependencies": { "@babel/core": "7.22.0", diff --git a/package.json b/package.json index d4e59048..d899299e 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "type": "git", "url": "https://github.com/mkkellogg/GaussianSplats3D" }, - "version": "0.4.3", + "version": "0.4.4", "description": "Three.js-based 3D Gaussian splat viewer", "module": "build/gaussian-splats-3d.module.js", "main": "build/gaussian-splats-3d.umd.cjs", diff --git a/src/AbortablePromise.js b/src/AbortablePromise.js index 429a1ff7..0c097075 100644 --- a/src/AbortablePromise.js +++ b/src/AbortablePromise.js @@ -64,8 +64,8 @@ export class AbortablePromise { }, this.abortHandler); } - abort() { - if (this.abortHandler) this.abortHandler(); + abort(reason) { + if (this.abortHandler) this.abortHandler(reason); } } diff --git a/src/DropInViewer.js b/src/DropInViewer.js index 2c5573fa..b56c2c94 100644 --- a/src/DropInViewer.js +++ b/src/DropInViewer.js @@ -108,6 +108,10 @@ export class DropInViewer extends THREE.Group { return this.viewer.removeSplatScenes(indexes, showLoadingUI); } + getSceneCount() { + return this.viewer.getSceneCount(); + } + dispose() { return this.viewer.dispose(); } diff --git a/src/Util.js b/src/Util.js index ea2afee5..4e0a423b 100644 --- a/src/Util.js +++ b/src/Util.js @@ -59,15 +59,12 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { const abortController = new AbortController(); const signal = abortController.signal; let aborted = false; - let rejectFunc = null; const abortHandler = (reason) => { - abortController.abort(reason); - rejectFunc(new AbortedPromiseError('Fetch aborted.')); + abortController.abort(new AbortedPromiseError(reason)); aborted = true; }; return new AbortablePromise((resolve, reject) => { - rejectFunc = reject; fetch(path, { signal }) .then(async (data) => { const reader = data.body.getReader(); @@ -109,6 +106,9 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { break; } } + }) + .catch((error) => { + reject(error); }); }, abortHandler); diff --git a/src/Viewer.js b/src/Viewer.js index 69d903a3..244cd914 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -18,7 +18,7 @@ import { SceneFormat } from './loaders/SceneFormat.js'; import { WebXRMode } from './webxr/WebXRMode.js'; import { VRButton } from './webxr/VRButton.js'; import { ARButton } from './webxr/ARButton.js'; -import { delayedExecute, nativePromiseWithExtractedComponents, abortablePromiseWithExtractedComponents } from './Util.js'; +import { delayedExecute, abortablePromiseWithExtractedComponents } from './Util.js'; import { LoaderStatus } from './loaders/LoaderStatus.js'; import { RenderMode } from './RenderMode.js'; import { LogLevel } from './LogLevel.js'; @@ -626,6 +626,7 @@ export class Viewer { if (!this.splatMesh) return; const splatCount = this.splatMesh.getSplatCount(); if (splatCount > 0) { + this.splatMesh.updateVisibleRegionFadeDistance(this.sceneRevealMode); this.splatMesh.updateTransforms(); this.getRenderDimensions(renderDimensions); const focalLengthX = this.camera.projectionMatrix.elements[0] * 0.5 * @@ -809,10 +810,11 @@ export class Viewer { */ downloadAndBuildSingleSplatSceneStandardLoad(path, format, splatAlphaRemovalThreshold, buildFunc, onProgress, onException) { - const downloadAndBuildPromise = nativePromiseWithExtractedComponents(); + const downloadPromise = this.downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold, + onProgress, false, undefined, format); + const downloadAndBuildPromise = abortablePromiseWithExtractedComponents(downloadPromise.abortHandler); - const downloadPromise = this.downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold, onProgress, false, undefined, format) - .then((splatBuffer) => { + downloadPromise.then((splatBuffer) => { this.removeSplatSceneDownloadPromise(downloadPromise); return buildFunc(splatBuffer, true, true).then(() => { downloadAndBuildPromise.resolve(); @@ -821,12 +823,10 @@ export class Viewer { }) .catch((e) => { if (onException) onException(); - downloadAndBuildPromise.reject(); this.clearSplatSceneDownloadAndBuildPromise(); this.removeSplatSceneDownloadPromise(downloadPromise); - if (!(e instanceof AbortedPromiseError)) { - throw (new Error(`Viewer::addSplatScene -> Could not load file ${path}`)); - } + const error = (e instanceof AbortedPromiseError) ? e : new Error(`Viewer::addSplatScene -> Could not load file ${path}`); + downloadAndBuildPromise.reject(error); }); this.addSplatSceneDownloadPromise(downloadPromise); @@ -906,11 +906,9 @@ export class Viewer { .catch((e) => { this.clearSplatSceneDownloadAndBuildPromise(); this.removeSplatSceneDownloadPromise(splatSceneDownloadPromise); - if (!(e instanceof AbortedPromiseError)) { - splatSceneDownloadAndBuildPromise.reject(e); - if (progressiveLoadFirstSectionBuildPromise.reject) progressiveLoadFirstSectionBuildPromise.reject(e); - if (onDownloadException) onDownloadException(e); - } + const error = (e instanceof AbortedPromiseError) ? e : new Error(`Viewer::addSplatScene -> Could not load one or more scenes`); + progressiveLoadFirstSectionBuildPromise.reject(error); + if (onDownloadException) onDownloadException(error); }); return progressiveLoadFirstSectionBuildPromise.promise; @@ -969,22 +967,19 @@ export class Viewer { if (onProgress) onProgress(totalPercent, percentLabel, loaderStatus); }; - const downloadPromises = []; - const nativeLoadPromises = []; - const abortHandlers = []; + const baseDownloadPromises = []; + const nativeDownloadPromises = []; for (let i = 0; i < sceneOptions.length; i++) { const options = sceneOptions[i]; const format = (options.format !== undefined && options.format !== null) ? options.format : sceneFormatFromPath(options.path); - const downloadPromise = this.downloadSplatSceneToSplatBuffer(options.path, options.splatAlphaRemovalThreshold, - onLoadProgress.bind(this, i), false, undefined, format); - abortHandlers.push(downloadPromise.abortHandler); - downloadPromises.push(downloadPromise); - nativeLoadPromises.push(downloadPromise.promise); - this.addSplatSceneDownloadPromise(downloadPromise); + const baseDownloadPromise = this.downloadSplatSceneToSplatBuffer(options.path, options.splatAlphaRemovalThreshold, + onLoadProgress.bind(this, i), false, undefined, format); + baseDownloadPromises.push(baseDownloadPromise); + nativeDownloadPromises.push(baseDownloadPromise.promise); } - const downloadPromise = new AbortablePromise((resolve, reject) => { - Promise.all(nativeLoadPromises) + const downloadAndBuildPromise = new AbortablePromise((resolve, reject) => { + Promise.all(nativeDownloadPromises) .then((splatBuffers) => { if (showLoadingUI) this.loadingSpinner.removeTask(loadingUITaskId); if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); @@ -997,22 +992,21 @@ export class Viewer { .catch((e) => { if (showLoadingUI) this.loadingSpinner.removeTask(loadingUITaskId); this.clearSplatSceneDownloadAndBuildPromise(); - if (!(e instanceof AbortedPromiseError)) { - reject(new Error(`Viewer::addSplatScenes -> Could not load one or more splat scenes.`)); - } else { - resolve(); - } + const error = (e instanceof AbortedPromiseError) ? e : + new Error(`Viewer::addSplatScenes -> Could not load one or more splat scenes.`); + reject(error); }) .finally(() => { - for (let downloadPromise of downloadPromises) { - this.removeSplatSceneDownloadPromise(downloadPromise); - } + this.removeSplatSceneDownloadPromise(downloadAndBuildPromise); }); - }, () => { - for (let abortHandler of abortHandlers) abortHandler(); + }, (reason) => { + for (let baseDownloadPromise of baseDownloadPromises) { + baseDownloadPromise.abort(reason); + } }); - this.setSplatSceneDownloadAndBuildPromise(downloadPromise); - return downloadPromise; + this.addSplatSceneDownloadPromise(downloadAndBuildPromise); + this.setSplatSceneDownloadAndBuildPromise(downloadAndBuildPromise); + return downloadAndBuildPromise; } /** @@ -1085,25 +1079,26 @@ export class Viewer { }); } - this.updateSplatSort(true); - - if (!this.sortWorker) { - this.splatRenderReady = true; - removeSplatProcessingTask(); - resolver(); - } else { - if (enableRenderBeforeFirstSort) { + this.runSplatSort(true).then((sortRunning) => { + if (!this.sortWorker || !sortRunning) { this.splatRenderReady = true; + removeSplatProcessingTask(); + resolver(); } else { - this.runAfterNextSort.push(() => { + if (enableRenderBeforeFirstSort) { this.splatRenderReady = true; + } else { + this.runAfterNextSort.push(() => { + this.splatRenderReady = true; + }); + } + this.runAfterNextSort.push(() => { + removeSplatProcessingTask(); + resolver(); }); } - this.runAfterNextSort.push(() => { - removeSplatProcessingTask(); - resolver(); - }); - } + }); + }; return new Promise((resolve) => { @@ -1358,7 +1353,8 @@ export class Viewer { }); this.splatMesh.updateTransforms(); this.splatRenderReady = false; - this.updateSplatSort(true) + + this.runSplatSort(true) .then(() => { if (checkForEarlyExit()) { this.splatRenderReady = true; @@ -1477,6 +1473,7 @@ export class Viewer { this.sortWorkerTransforms = null; this.disposed = true; this.disposing = false; + this.disposePromise = null; }); promisesToAbort.forEach((toAbort) => { toAbort.abort('Scene disposed'); @@ -1573,8 +1570,7 @@ export class Viewer { Viewer.setCameraPositionFromZoom(this.camera, this.camera, this.controls); } } - this.splatMesh.updateVisibleRegionFadeDistance(this.sceneRevealMode); - this.updateSplatSort(); + this.runSplatSort(); this.updateForRendererSizeChanges(); this.updateSplatMesh(); this.updateMeshCursor(); @@ -1771,7 +1767,7 @@ export class Viewer { } } - updateSplatSort = function() { + runSplatSort = function() { const mvpMatrix = new THREE.Matrix4(); const cameraPositionArray = []; @@ -1796,11 +1792,12 @@ export class Viewer { } ]; - return async function(force = false) { - if (this.sortRunning) return; + return function(force = false) { + if (!this.initialized) return Promise.resolve(false); + if (this.sortRunning) return Promise.resolve(true); if (this.splatMesh.getSplatCount() <= 0) { this.splatRenderCount = 0; - return; + return false; } let angleDiff = 0; @@ -1816,7 +1813,7 @@ export class Viewer { if (!this.splatMesh.dynamicMode && queuedSorts.length === 0) { if (angleDiff <= 0.99) needsRefreshForRotation = true; if (positionDiff >= 1.0) needsRefreshForPosition = true; - if (!needsRefreshForRotation && !needsRefreshForPosition) return; + if (!needsRefreshForRotation && !needsRefreshForPosition) return Promise.resolve(false); } } @@ -1829,61 +1826,66 @@ export class Viewer { mvpMatrix.premultiply(mvpCamera.projectionMatrix); mvpMatrix.multiply(this.splatMesh.matrixWorld); + let gpuAcceleratedSortPromise = Promise.resolve(); if (this.gpuAcceleratedSort && (queuedSorts.length <= 1 || queuedSorts.length % 2 === 0)) { - await this.splatMesh.computeDistancesOnGPU(mvpMatrix, this.sortWorkerPrecomputedDistances); + gpuAcceleratedSortPromise = this.splatMesh.computeDistancesOnGPU(mvpMatrix, this.sortWorkerPrecomputedDistances); } - if (this.splatMesh.dynamicMode || shouldSortAll) { - queuedSorts.push(this.splatRenderCount); - } else { - if (queuedSorts.length === 0) { - for (let partialSort of partialSorts) { - if (angleDiff < partialSort.angleThreshold) { - for (let sortFraction of partialSort.sortFractions) { - queuedSorts.push(Math.floor(this.splatRenderCount * sortFraction)); + gpuAcceleratedSortPromise.then(() => { + if (this.splatMesh.dynamicMode || shouldSortAll) { + queuedSorts.push(this.splatRenderCount); + } else { + if (queuedSorts.length === 0) { + for (let partialSort of partialSorts) { + if (angleDiff < partialSort.angleThreshold) { + for (let sortFraction of partialSort.sortFractions) { + queuedSorts.push(Math.floor(this.splatRenderCount * sortFraction)); + } + break; } - break; } + queuedSorts.push(this.splatRenderCount); } - queuedSorts.push(this.splatRenderCount); } - } - let sortCount = Math.min(queuedSorts.shift(), this.splatRenderCount); - - cameraPositionArray[0] = this.camera.position.x; - cameraPositionArray[1] = this.camera.position.y; - cameraPositionArray[2] = this.camera.position.z; - - const sortMessage = { - 'modelViewProj': mvpMatrix.elements, - 'cameraPosition': cameraPositionArray, - 'splatRenderCount': this.splatRenderCount, - 'splatSortCount': sortCount, - 'usePrecomputedDistances': this.gpuAcceleratedSort - }; - if (this.splatMesh.dynamicMode) { - this.splatMesh.fillTransformsArray(this.sortWorkerTransforms); - } - if (!this.sharedMemoryForWorkers) { - sortMessage.indexesToSort = this.sortWorkerIndexesToSort; - sortMessage.transforms = this.sortWorkerTransforms; - if (this.gpuAcceleratedSort) { - sortMessage.precomputedDistances = this.sortWorkerPrecomputedDistances; + let sortCount = Math.min(queuedSorts.shift(), this.splatRenderCount); + + cameraPositionArray[0] = this.camera.position.x; + cameraPositionArray[1] = this.camera.position.y; + cameraPositionArray[2] = this.camera.position.z; + + const sortMessage = { + 'modelViewProj': mvpMatrix.elements, + 'cameraPosition': cameraPositionArray, + 'splatRenderCount': this.splatRenderCount, + 'splatSortCount': sortCount, + 'usePrecomputedDistances': this.gpuAcceleratedSort + }; + if (this.splatMesh.dynamicMode) { + this.splatMesh.fillTransformsArray(this.sortWorkerTransforms); + } + if (!this.sharedMemoryForWorkers) { + sortMessage.indexesToSort = this.sortWorkerIndexesToSort; + sortMessage.transforms = this.sortWorkerTransforms; + if (this.gpuAcceleratedSort) { + sortMessage.precomputedDistances = this.sortWorkerPrecomputedDistances; + } } - } - this.sortPromise = new Promise((resolve) => { - this.sortPromiseResolver = resolve; - }); + this.sortPromise = new Promise((resolve) => { + this.sortPromiseResolver = resolve; + }); + + this.sortWorker.postMessage({ + 'sort': sortMessage + }); - this.sortWorker.postMessage({ - 'sort': sortMessage + if (queuedSorts.length === 0) { + lastSortViewPos.copy(this.camera.position); + lastSortViewDir.copy(sortViewDir); + } }); - if (queuedSorts.length === 0) { - lastSortViewPos.copy(this.camera.position); - lastSortViewDir.copy(sortViewDir); - } + return gpuAcceleratedSortPromise; }; }(); @@ -2014,6 +2016,10 @@ export class Viewer { return this.splatMesh.getScene(sceneIndex); } + getSceneCount() { + return this.splatMesh.getSceneCount(); + } + isMobile() { return navigator.userAgent.includes('Mobi'); } diff --git a/src/loaders/SplatBuffer.js b/src/loaders/SplatBuffer.js index 4ec7cf4c..9a1d19a4 100644 --- a/src/loaders/SplatBuffer.js +++ b/src/loaders/SplatBuffer.js @@ -3,36 +3,39 @@ import { UncompressedSplatArray } from './UncompressedSplatArray.js'; import { clamp, getSphericalHarmonicsComponentCountForDegree } from '../Util.js'; import { Constants } from '../Constants.js'; -const SphericalHarmonics8BitCompressionHalfRange = Constants.SphericalHarmonics8BitCompressionRange / 2.0; +const DefaultSphericalHarmonics8BitCompressionRange = Constants.SphericalHarmonics8BitCompressionRange; +const DefaultSphericalHarmonics8BitCompressionHalfRange = DefaultSphericalHarmonics8BitCompressionRange / 2.0; const toHalfFloat = THREE.DataUtils.toHalfFloat.bind(THREE.DataUtils); const fromHalfFloat = THREE.DataUtils.fromHalfFloat.bind(THREE.DataUtils); -const toUncompressedFloat = (f, compressionLevel, isSH = false) => { +const toUncompressedFloat = (f, compressionLevel, isSH = false, range8BitMin, range8BitMax) => { if (compressionLevel === 0) { return f; } else if (compressionLevel === 1 || compressionLevel === 2 && !isSH) { return THREE.DataUtils.fromHalfFloat(f); } else if (compressionLevel === 2) { - return fromUint8(f); + return fromUint8(f, range8BitMin, range8BitMax); } }; -const toUint8 = (v) => { - v = clamp(v, -SphericalHarmonics8BitCompressionHalfRange, SphericalHarmonics8BitCompressionHalfRange); - return clamp(Math.floor((v * (0.5 / SphericalHarmonics8BitCompressionHalfRange) + 0.5) * 255), 0, 255); +const toUint8 = (v, rangeMin, rangeMax) => { + v = clamp(v, rangeMin, rangeMax); + const range = (rangeMax - rangeMin); + return clamp(Math.floor((v - rangeMin) / range * 255), 0, 255); }; -const fromUint8 = (v) => { - return (v / 255) * Constants.SphericalHarmonics8BitCompressionRange - SphericalHarmonics8BitCompressionHalfRange; +const fromUint8 = (v, rangeMin, rangeMax) => { + const range = (rangeMax - rangeMin); + return (v / 255 * range + rangeMin); }; -const fromHalfFloatToUint8 = (v) => { - return toUint8(fromHalfFloat(v)); +const fromHalfFloatToUint8 = (v, rangeMin, rangeMax) => { + return toUint8(fromHalfFloat(v, rangeMin, rangeMax)); }; -const fromUint8ToHalfFloat = (v) => { - return toHalfFloat(fromUint8(v)); +const fromUint8ToHalfFloat = (v, rangeMin, rangeMax) => { + return toHalfFloat(fromUint8(v, rangeMin, rangeMax)); }; const dataViewFloatForCompressionLevel = (dataView, floatIndex, compressionLevel, isSH = false) => { @@ -607,10 +610,10 @@ export class SplatBuffer { destArray[destBase + 2] = conversionFunc(srcArray[2]); }; - const toUncompressedFloatArray3 = (src, dest, compressionLevel) => { - dest[0] = toUncompressedFloat(src[0], compressionLevel, true); - dest[1] = toUncompressedFloat(src[1], compressionLevel, true); - dest[2] = toUncompressedFloat(src[2], compressionLevel, true); + const toUncompressedFloatArray3 = (src, dest, compressionLevel, range8BitMin, range8BitMax) => { + dest[0] = toUncompressedFloat(src[0], compressionLevel, true, range8BitMin, range8BitMax); + dest[1] = toUncompressedFloat(src[1], compressionLevel, true, range8BitMin, range8BitMax); + dest[2] = toUncompressedFloat(src[2], compressionLevel, true, range8BitMin, range8BitMax); return dest; }; @@ -633,6 +636,14 @@ export class SplatBuffer { set3(sh13, tempMatrix3.elements[3], -tempMatrix3.elements[6], tempMatrix3.elements[0]); } + const localFromHalfFloatToUint8 = (v) => { + return fromHalfFloatToUint8(v, this.minSphericalHarmonicsCoeff, this.maxSphericalHarmonicsCoeff); + }; + + const localToUint8 = (v) => { + return toUint8(v, this.minSphericalHarmonicsCoeff, this.maxSphericalHarmonicsCoeff); + }; + for (let i = srcFrom; i <= srcTo; i++) { const sectionIndex = this.globalSplatIndexToSectionMap[i]; @@ -654,13 +665,16 @@ export class SplatBuffer { if (compressionLevelForOutputConversion !== desiredOutputCompressionLevel) { if (compressionLevelForOutputConversion === 1) { if (desiredOutputCompressionLevel === 0) outputConversionFunc = fromHalfFloat; - else if (desiredOutputCompressionLevel == 2) outputConversionFunc = fromHalfFloatToUint8; + else if (desiredOutputCompressionLevel == 2) outputConversionFunc = localFromHalfFloatToUint8; } else if (compressionLevelForOutputConversion === 0) { if (desiredOutputCompressionLevel === 1) outputConversionFunc = toHalfFloat; - else if (desiredOutputCompressionLevel == 2) outputConversionFunc = toUint8; + else if (desiredOutputCompressionLevel == 2) outputConversionFunc = localToUint8; } } + const minShCoeff = this.minSphericalHarmonicsCoeff; + const maxShCoeff = this.maxSphericalHarmonicsCoeff; + if (outSphericalHarmonicsDegree >= 1) { set3FromArray(shIn1, dataView, 3, 0, this.compressionLevel); @@ -668,9 +682,9 @@ export class SplatBuffer { set3FromArray(shIn3, dataView, 3, 2, this.compressionLevel); if (transform) { - toUncompressedFloatArray3(shIn1, shIn1, this.compressionLevel); - toUncompressedFloatArray3(shIn2, shIn2, this.compressionLevel); - toUncompressedFloatArray3(shIn3, shIn3, this.compressionLevel); + toUncompressedFloatArray3(shIn1, shIn1, this.compressionLevel, minShCoeff, maxShCoeff); + toUncompressedFloatArray3(shIn2, shIn2, this.compressionLevel, minShCoeff, maxShCoeff); + toUncompressedFloatArray3(shIn3, shIn3, this.compressionLevel, minShCoeff, maxShCoeff); SplatBuffer.rotateSphericalHarmonics3(shIn1, shIn2, shIn3, sh11, sh12, sh13, shOut1, shOut2, shOut3); } else { copy3(shIn1, shOut1); @@ -691,11 +705,11 @@ export class SplatBuffer { set3FromArray(shIn5, dataView, 5, 13, this.compressionLevel); if (transform) { - toUncompressedFloatArray3(shIn1, shIn1, this.compressionLevel); - toUncompressedFloatArray3(shIn2, shIn2, this.compressionLevel); - toUncompressedFloatArray3(shIn3, shIn3, this.compressionLevel); - toUncompressedFloatArray3(shIn4, shIn4, this.compressionLevel); - toUncompressedFloatArray3(shIn5, shIn5, this.compressionLevel); + toUncompressedFloatArray3(shIn1, shIn1, this.compressionLevel, minShCoeff, maxShCoeff); + toUncompressedFloatArray3(shIn2, shIn2, this.compressionLevel, minShCoeff, maxShCoeff); + toUncompressedFloatArray3(shIn3, shIn3, this.compressionLevel, minShCoeff, maxShCoeff); + toUncompressedFloatArray3(shIn4, shIn4, this.compressionLevel, minShCoeff, maxShCoeff); + toUncompressedFloatArray3(shIn5, shIn5, this.compressionLevel, minShCoeff, maxShCoeff); SplatBuffer.rotateSphericalHarmonics5(shIn1, shIn2, shIn3, shIn4, shIn5, sh11, sh12, sh13, sh21, sh22, sh23, sh24, sh25, shOut1, shOut2, shOut3, shOut4, shOut5); @@ -816,6 +830,9 @@ export class SplatBuffer { const compressionLevel = headerArrayUint16[10]; const sceneCenter = new THREE.Vector3(headerArrayFloat32[6], headerArrayFloat32[7], headerArrayFloat32[8]); + const minSphericalHarmonicsCoeff = headerArrayFloat32[9] || -DefaultSphericalHarmonics8BitCompressionHalfRange; + const maxSphericalHarmonicsCoeff = headerArrayFloat32[10] || DefaultSphericalHarmonics8BitCompressionHalfRange; + return { versionMajor, versionMinor, @@ -824,7 +841,9 @@ export class SplatBuffer { maxSplatCount, splatCount, compressionLevel, - sceneCenter + sceneCenter, + minSphericalHarmonicsCoeff, + maxSphericalHarmonicsCoeff }; } @@ -851,6 +870,8 @@ export class SplatBuffer { headerArrayFloat32[6] = header.sceneCenter.x; headerArrayFloat32[7] = header.sceneCenter.y; headerArrayFloat32[8] = header.sceneCenter.z; + headerArrayFloat32[9] = header.minSphericalHarmonicsCoeff || -DefaultSphericalHarmonics8BitCompressionHalfRange; + headerArrayFloat32[10] = header.maxSphericalHarmonicsCoeff || DefaultSphericalHarmonics8BitCompressionHalfRange; } static parseSectionHeaders(header, buffer, offset = 0, secLoadedCountsToMax) { @@ -959,6 +980,8 @@ export class SplatBuffer { this.splatCount = secLoadedCountsToMax ? header.maxSplatCount : 0; this.compressionLevel = header.compressionLevel; this.sceneCenter = new THREE.Vector3().copy(header.sceneCenter); + this.minSphericalHarmonicsCoeff = header.minSphericalHarmonicsCoeff; + this.maxSphericalHarmonicsCoeff = header.maxSphericalHarmonicsCoeff; this.sections = SplatBuffer.parseSectionHeaders(header, this.bufferData, SplatBuffer.HeaderSizeBytes, secLoadedCountsToMax); @@ -1050,7 +1073,9 @@ export class SplatBuffer { }; return function(targetSplat, sectionBuffer, bufferOffset, compressionLevel, sphericalHarmonicsDegree, - bucketCenter, compressionScaleFactor, compressionScaleRange) { + bucketCenter, compressionScaleFactor, compressionScaleRange, + minSphericalHarmonicsCoeff = -DefaultSphericalHarmonics8BitCompressionHalfRange, + maxSphericalHarmonicsCoeff = DefaultSphericalHarmonics8BitCompressionHalfRange) { const sphericalHarmonicsComponentsPerSplat = getSphericalHarmonicsComponentCountForDegree(sphericalHarmonicsDegree); const bytesPerCenter = SplatBuffer.CompressionLevels[compressionLevel].BytesPerCenter; @@ -1118,14 +1143,16 @@ export class SplatBuffer { if (sphericalHarmonicsDegree >= 1) { for (let s = 0; s < 9; s++) { const srcVal = targetSplat[OFFSET_FRC0 + s] || 0; - shOut[s] = compressionLevel === 1 ? toHalfFloat(srcVal) : toUint8(srcVal); + shOut[s] = compressionLevel === 1 ? toHalfFloat(srcVal) : + toUint8(srcVal, minSphericalHarmonicsCoeff, maxSphericalHarmonicsCoeff); } const degree1ByteCount = 9 * bytesPerSHComponent; copyBetweenBuffers(shOut.buffer, 0, sectionBuffer, sphericalHarmonicsBase, degree1ByteCount); if (sphericalHarmonicsDegree >= 2) { for (let s = 0; s < 15; s++) { const srcVal = targetSplat[OFFSET_FRC9 + s] || 0; - shOut[s + 9] = compressionLevel === 1 ? toHalfFloat(srcVal) : toUint8(srcVal); + shOut[s + 9] = compressionLevel === 1 ? toHalfFloat(srcVal) : + toUint8(srcVal, minSphericalHarmonicsCoeff, maxSphericalHarmonicsCoeff); } copyBetweenBuffers(shOut.buffer, degree1ByteCount, sectionBuffer, sphericalHarmonicsBase + degree1ByteCount, 15 * bytesPerSHComponent); @@ -1156,6 +1183,27 @@ export class SplatBuffer { shDegree = Math.max(splatArray.sphericalHarmonicsDegree, shDegree); } + let minSphericalHarmonicsCoeff; + let maxSphericalHarmonicsCoeff; + + for (let sa = 0; sa < splatArrays.length; sa ++) { + const splatArray = splatArrays[sa]; + for (let i = 0; i < splatArray.splats.length; i++) { + const splat = splatArray.splats[i]; + for (let sc = UncompressedSplatArray.OFFSET.FRC0; sc < UncompressedSplatArray.OFFSET.FRC23 && sc < splat.length; sc++) { + if (!minSphericalHarmonicsCoeff || splat[sc] < minSphericalHarmonicsCoeff) { + minSphericalHarmonicsCoeff = splat[sc]; + } + if (!maxSphericalHarmonicsCoeff || splat[sc] > maxSphericalHarmonicsCoeff) { + maxSphericalHarmonicsCoeff = splat[sc]; + } + } + } + } + + minSphericalHarmonicsCoeff = minSphericalHarmonicsCoeff || -DefaultSphericalHarmonics8BitCompressionHalfRange; + maxSphericalHarmonicsCoeff = maxSphericalHarmonicsCoeff || DefaultSphericalHarmonics8BitCompressionHalfRange; + const { bytesPerSplat } = SplatBuffer.calculateComponentStorage(compressionLevel, shDegree); const compressionScaleRange = SplatBuffer.CompressionLevels[compressionLevel].ScaleRange; @@ -1202,7 +1250,8 @@ export class SplatBuffer { const targetSplat = validSplats.splats[row]; const bufferOffset = bucketDataBytes + outSplatCount * bytesPerSplat; SplatBuffer.writeSplatDataToSectionBuffer(targetSplat, sectionBuffer, bufferOffset, compressionLevel, shDegree, - bucketCenter, compressionScaleFactor, compressionScaleRange); + bucketCenter, compressionScaleFactor, compressionScaleRange, + minSphericalHarmonicsCoeff, maxSphericalHarmonicsCoeff); outSplatCount++; } } @@ -1256,7 +1305,9 @@ export class SplatBuffer { maxSplatCount: totalSplatCount, splatCount: totalSplatCount, compressionLevel: compressionLevel, - sceneCenter: sceneCenter + sceneCenter: sceneCenter, + minSphericalHarmonicsCoeff: minSphericalHarmonicsCoeff, + maxSphericalHarmonicsCoeff: maxSphericalHarmonicsCoeff }, unifiedBuffer); let currentUnifiedBase = SplatBuffer.HeaderSizeBytes; diff --git a/src/splatmesh/SplatMaterial.js b/src/splatmesh/SplatMaterial.js index da3aaab0..fde5c4a5 100644 --- a/src/splatmesh/SplatMaterial.js +++ b/src/splatmesh/SplatMaterial.js @@ -14,14 +14,11 @@ export class SplatMaterial { uniform highp sampler2D sphericalHarmonicsTextureR; uniform highp sampler2D sphericalHarmonicsTextureG; uniform highp sampler2D sphericalHarmonicsTextureB; - `; - if (enableOptionalEffects || dynamicMode) { - vertexShaderSource += ` - uniform highp usampler2D sceneIndexesTexture; - uniform vec2 sceneIndexesTextureSize; - `; - } + uniform highp usampler2D sceneIndexesTexture; + uniform vec2 sceneIndexesTextureSize; + uniform int sceneCount; + `; if (enableOptionalEffects) { vertexShaderSource += ` @@ -57,6 +54,8 @@ export class SplatMaterial { uniform int fadeInComplete; uniform vec3 sceneCenter; uniform float splatScale; + uniform float sphericalHarmonics8BitCompressionRangeMin[${Constants.MaxScenes}]; + uniform float sphericalHarmonics8BitCompressionRangeMax[${Constants.MaxScenes}]; varying vec4 vColor; varying vec2 vUv; @@ -110,10 +109,6 @@ export class SplatMaterial { const float SH_C1 = 0.4886025119029199f; const float[5] SH_C2 = float[](1.0925484, -1.0925484, 0.3153916, -1.0925484, 0.5462742); - const float SphericalHarmonics8BitCompressionRange = ${Constants.SphericalHarmonics8BitCompressionRange.toFixed(1)}; - const float SphericalHarmonics8BitCompressionHalfRange = SphericalHarmonics8BitCompressionRange / 2.0; - const vec3 vec8BitSHShift = vec3(SphericalHarmonics8BitCompressionHalfRange); - void main () { uint oddOffset = splatIndex & uint(0x00000001); @@ -123,13 +118,13 @@ export class SplatMaterial { float fOddOffset = float(oddOffset); uvec4 sampledCenterColor = texture(centersColorsTexture, getDataUV(1, 0, centersColorsTextureSize)); - vec3 splatCenter = uintBitsToFloat(uvec3(sampledCenterColor.gba));`; + vec3 splatCenter = uintBitsToFloat(uvec3(sampledCenterColor.gba)); - if (dynamicMode || enableOptionalEffects) { - vertexShaderSource += ` - uint sceneIndex = texture(sceneIndexesTexture, getDataUV(1, 0, sceneIndexesTextureSize)).r; + uint sceneIndex = uint(0); + if (sceneCount > 1) { + sceneIndex = texture(sceneIndexesTexture, getDataUV(1, 0, sceneIndexesTextureSize)).r; + } `; - } if (enableOptionalEffects) { vertexShaderSource += ` @@ -152,6 +147,12 @@ export class SplatMaterial { } vertexShaderSource += ` + float sh8BitCompressionRangeMinForScene = sphericalHarmonics8BitCompressionRangeMin[sceneIndex]; + float sh8BitCompressionRangeMaxForScene = sphericalHarmonics8BitCompressionRangeMax[sceneIndex]; + float sh8BitCompressionRangeForScene = sh8BitCompressionRangeMaxForScene - sh8BitCompressionRangeMinForScene; + float sh8BitCompressionHalfRangeForScene = sh8BitCompressionRangeForScene / 2.0; + vec3 vec8BitSHShift = vec3(sh8BitCompressionRangeMinForScene); + vec4 viewCenter = transformModelViewMatrix * vec4(splatCenter, 1.0); vec4 clipCenter = projectionMatrix * viewCenter; @@ -168,6 +169,7 @@ export class SplatMaterial { vColor = uintToRGBAVec(sampledCenterColor.r); `; + // Proceed to sampling and rendering 1st degree spherical harmonics if (maxSphericalHarmonicsDegree >= 1) { vertexShaderSource += ` @@ -176,8 +178,7 @@ export class SplatMaterial { if (dynamicMode) { vertexShaderSource += ` - mat4 mTransform = modelMatrix * transform; - vec3 worldViewDir = normalize(splatCenter - vec3(inverse(mTransform) * vec4(cameraPosition, 1.0))); + vec3 worldViewDir = normalize(splatCenter - vec3(inverse(transform) * vec4(cameraPosition, 1.0))); `; } else { vertexShaderSource += ` @@ -193,31 +194,20 @@ export class SplatMaterial { if (maxSphericalHarmonicsDegree >= 2) { vertexShaderSource += ` - vec4 sampledSH0123; - vec4 sampledSH4567; - vec4 sampledSH891011; - - vec4 sampledSH0123R; - vec4 sampledSH0123G; - vec4 sampledSH0123B; - - if (sphericalHarmonicsMultiTextureMode == 0) { - sampledSH0123 = texture(sphericalHarmonicsTexture, getDataUV(6, 0, sphericalHarmonicsTextureSize)); - sampledSH4567 = texture(sphericalHarmonicsTexture, getDataUV(6, 1, sphericalHarmonicsTextureSize)); - sampledSH891011 = texture(sphericalHarmonicsTexture, getDataUV(6, 2, sphericalHarmonicsTextureSize)); - sh1 = sampledSH0123.rgb; - sh2 = vec3(sampledSH0123.a, sampledSH4567.rg); - sh3 = vec3(sampledSH4567.ba, sampledSH891011.r); - } else { - sampledSH0123R = texture(sphericalHarmonicsTextureR, getDataUV(2, 0, sphericalHarmonicsTextureSize)); - sampledSH0123G = texture(sphericalHarmonicsTextureG, getDataUV(2, 0, sphericalHarmonicsTextureSize)); - sampledSH0123B = texture(sphericalHarmonicsTextureB, getDataUV(2, 0, sphericalHarmonicsTextureSize)); - sh1 = vec3(sampledSH0123R.rgb); - sh2 = vec3(sampledSH0123G.rgb); - sh3 = vec3(sampledSH0123B.rgb); - } + vec3 sh4; + vec3 sh5; + vec3 sh6; + vec3 sh7; + vec3 sh8; `; - } else { + } + + // Determining how to sample spherical harmonics textures to get the coefficients for calculations for a given degree + // depends on how many total degrees (maxSphericalHarmonicsDegree) are present in the textures. This is because that + // number affects how they are packed in the textures, and therefore the offset & stride required to access them. + + // Sample spherical harmonics textures with 1 degree worth of data for 1st degree calculations, and store in sh1, sh2, and sh3 + if (maxSphericalHarmonicsDegree === 1) { vertexShaderSource += ` if (sphericalHarmonicsMultiTextureMode == 0) { vec2 shUV = getDataUVF(nearestEvenIndex, 2.5, doubleOddOffset, sphericalHarmonicsTextureSize); @@ -241,13 +231,41 @@ export class SplatMaterial { sh3 = vec3(sampledSH01B.rg, sampledSH23B.r); } `; + // Sample spherical harmonics textures with 2 degrees worth of data for 1st degree calculations, and store in sh1, sh2, and sh3 + } else if (maxSphericalHarmonicsDegree === 2) { + vertexShaderSource += ` + vec4 sampledSH0123; + vec4 sampledSH4567; + vec4 sampledSH891011; + + vec4 sampledSH0123R; + vec4 sampledSH0123G; + vec4 sampledSH0123B; + + if (sphericalHarmonicsMultiTextureMode == 0) { + sampledSH0123 = texture(sphericalHarmonicsTexture, getDataUV(6, 0, sphericalHarmonicsTextureSize)); + sampledSH4567 = texture(sphericalHarmonicsTexture, getDataUV(6, 1, sphericalHarmonicsTextureSize)); + sampledSH891011 = texture(sphericalHarmonicsTexture, getDataUV(6, 2, sphericalHarmonicsTextureSize)); + sh1 = sampledSH0123.rgb; + sh2 = vec3(sampledSH0123.a, sampledSH4567.rg); + sh3 = vec3(sampledSH4567.ba, sampledSH891011.r); + } else { + sampledSH0123R = texture(sphericalHarmonicsTextureR, getDataUV(2, 0, sphericalHarmonicsTextureSize)); + sampledSH0123G = texture(sphericalHarmonicsTextureG, getDataUV(2, 0, sphericalHarmonicsTextureSize)); + sampledSH0123B = texture(sphericalHarmonicsTextureB, getDataUV(2, 0, sphericalHarmonicsTextureSize)); + sh1 = vec3(sampledSH0123R.rgb); + sh2 = vec3(sampledSH0123G.rgb); + sh3 = vec3(sampledSH0123B.rgb); + } + `; } + // Perform 1st degree spherical harmonics calculations vertexShaderSource += ` if (sphericalHarmonics8BitMode == 1) { - sh1 = sh1 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh2 = sh2 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh3 = sh3 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; + sh1 = sh1 * sh8BitCompressionRangeForScene + vec8BitSHShift; + sh2 = sh2 * sh8BitCompressionRangeForScene + vec8BitSHShift; + sh3 = sh3 * sh8BitCompressionRangeForScene + vec8BitSHShift; } float x = worldViewDir.x; float y = worldViewDir.y; @@ -255,6 +273,7 @@ export class SplatMaterial { vColor.rgb += SH_C1 * (-sh1 * y + sh2 * z - sh3 * x); `; + // Proceed to sampling and rendering 2nd degree spherical harmonics if (maxSphericalHarmonicsDegree >= 2) { vertexShaderSource += ` @@ -265,13 +284,12 @@ export class SplatMaterial { float xy = x * y; float yz = y * z; float xz = x * z; + `; - vec3 sh4; - vec3 sh5; - vec3 sh6; - vec3 sh7; - vec3 sh8; - + // Sample spherical harmonics textures with 2 degrees worth of data for 2nd degree calculations, + // and store in sh4, sh5, sh6, sh7, and sh8 + if (maxSphericalHarmonicsDegree === 2) { + vertexShaderSource += ` if (sphericalHarmonicsMultiTextureMode == 0) { vec4 sampledSH12131415 = texture(sphericalHarmonicsTexture, getDataUV(6, 3, sphericalHarmonicsTextureSize)); vec4 sampledSH16171819 = texture(sphericalHarmonicsTexture, getDataUV(6, 4, sphericalHarmonicsTextureSize)); @@ -291,13 +309,17 @@ export class SplatMaterial { sh7 = vec3(sampledSH4567G.a, sampledSH0123B.a, sampledSH4567B.r); sh8 = vec3(sampledSH4567B.gba); } + `; + } + // Perform 2nd degree spherical harmonics calculations + vertexShaderSource += ` if (sphericalHarmonics8BitMode == 1) { - sh4 = sh4 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh5 = sh5 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh6 = sh6 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh7 = sh7 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; - sh8 = sh8 * SphericalHarmonics8BitCompressionRange - vec8BitSHShift; + sh4 = sh4 * sh8BitCompressionRangeForScene + vec8BitSHShift; + sh5 = sh5 * sh8BitCompressionRangeForScene + vec8BitSHShift; + sh6 = sh6 * sh8BitCompressionRangeForScene + vec8BitSHShift; + sh7 = sh7 * sh8BitCompressionRangeForScene + vec8BitSHShift; + sh8 = sh8 * sh8BitCompressionRangeForScene + vec8BitSHShift; } vColor.rgb += @@ -311,7 +333,7 @@ export class SplatMaterial { } vertexShaderSource += ` - + vColor.rgb = clamp(vColor.rgb, vec3(0.), vec3(1.)); } @@ -392,6 +414,14 @@ export class SplatMaterial { 'type': 't', 'value': null }, + 'sphericalHarmonics8BitCompressionRangeMin': { + 'type': 'f', + 'value': [] + }, + 'sphericalHarmonics8BitCompressionRangeMax': { + 'type': 'f', + 'value': [] + }, 'focal': { 'type': 'v2', 'value': new THREE.Vector2() @@ -443,18 +473,23 @@ export class SplatMaterial { 'pointCloudModeEnabled': { 'type': 'i', 'value': pointCloudModeEnabled ? 1 : 0 - } - }; - - if (dynamicMode || enableOptionalEffects) { - uniforms['sceneIndexesTexture'] = { + }, + 'sceneIndexesTexture': { 'type': 't', 'value': null - }; - uniforms['sceneIndexesTextureSize'] = { + }, + 'sceneIndexesTextureSize': { 'type': 'v2', 'value': new THREE.Vector2(1024, 1024) - }; + }, + 'sceneCount': { + 'type': 'i', + 'value': 1 + } + }; + for (let i = 0; i < Constants.MaxScenes; i++) { + uniforms.sphericalHarmonics8BitCompressionRangeMin.value.push(-Constants.SphericalHarmonics8BitCompressionRange / 2.0); + uniforms.sphericalHarmonics8BitCompressionRangeMax.value.push(Constants.SphericalHarmonics8BitCompressionRange / 2.0); } if (enableOptionalEffects) { diff --git a/src/splatmesh/SplatMaterial2D.js b/src/splatmesh/SplatMaterial2D.js index 941c22c6..e8e04d58 100644 --- a/src/splatmesh/SplatMaterial2D.js +++ b/src/splatmesh/SplatMaterial2D.js @@ -58,6 +58,8 @@ export class SplatMaterial2D { } static buildVertexShaderProjection() { + + // Original CUDA code for calculating splat-to-screen transformation, for reference /* glm::mat3 R = quat_to_rotmat(rot); glm::mat3 S = scale_to_mat(scale, mod); @@ -85,7 +87,6 @@ export class SplatMaterial2D { T = glm::transpose(splat2world) * world2ndc * ndc2pix; normal = transformVec4x3({L[2].x, L[2].y, L[2].z}, viewmatrix); - */ // Compute a 2D-to-2D mapping matrix from a tangent plane into a image plane @@ -125,6 +126,7 @@ export class SplatMaterial2D { vec3 normal = vec3(viewMatrix * vec4(L[0][2], L[1][2], L[2][2], 0.0)); `; + // Original CUDA code for projection to 2D, for reference /* float3 T0 = {T[0][0], T[0][1], T[0][2]}; float3 T1 = {T[1][0], T[1][1], T[1][2]}; @@ -149,6 +151,7 @@ export class SplatMaterial2D { extent = sqrtf2(maxf2(1e-4, half_extend)); return true; */ + // Computing the bounding box of the 2D Gaussian and its center // The center of the bounding box is used to create a low pass filter. // This code is based off the reference implementation and creates an AABB aligned @@ -240,6 +243,49 @@ export class SplatMaterial2D { static buildFragmentShader() { + // Original CUDA code for splat intersection, for reference + /* + const float2 xy = collected_xy[j]; + const float3 Tu = collected_Tu[j]; + const float3 Tv = collected_Tv[j]; + const float3 Tw = collected_Tw[j]; + float3 k = pix.x * Tw - Tu; + float3 l = pix.y * Tw - Tv; + float3 p = cross(k, l); + if (p.z == 0.0) continue; + float2 s = {p.x / p.z, p.y / p.z}; + float rho3d = (s.x * s.x + s.y * s.y); + float2 d = {xy.x - pixf.x, xy.y - pixf.y}; + float rho2d = FilterInvSquare * (d.x * d.x + d.y * d.y); + + // compute intersection and depth + float rho = min(rho3d, rho2d); + float depth = (rho3d <= rho2d) ? (s.x * Tw.x + s.y * Tw.y) + Tw.z : Tw.z; + if (depth < near_n) continue; + float4 nor_o = collected_normal_opacity[j]; + float normal[3] = {nor_o.x, nor_o.y, nor_o.z}; + float opa = nor_o.w; + + float power = -0.5f * rho; + if (power > 0.0f) + continue; + + // Eq. (2) from 3D Gaussian splatting paper. + // Obtain alpha by multiplying with Gaussian opacity + // and its exponential falloff from mean. + // Avoid numerical instabilities (see paper appendix). + float alpha = min(0.99f, opa * exp(power)); + if (alpha < 1.0f / 255.0f) + continue; + float test_T = T * (1 - alpha); + if (test_T < 0.0001f) + { + done = true; + continue; + } + + float w = alpha * T; + */ let fragmentShaderSource = ` precision highp float; #include @@ -253,48 +299,6 @@ export class SplatMaterial2D { varying vec2 vQuadCenter; varying vec2 vFragCoord; - /* - const float2 xy = collected_xy[j]; - const float3 Tu = collected_Tu[j]; - const float3 Tv = collected_Tv[j]; - const float3 Tw = collected_Tw[j]; - float3 k = pix.x * Tw - Tu; - float3 l = pix.y * Tw - Tv; - float3 p = cross(k, l); - if (p.z == 0.0) continue; - float2 s = {p.x / p.z, p.y / p.z}; - float rho3d = (s.x * s.x + s.y * s.y); - float2 d = {xy.x - pixf.x, xy.y - pixf.y}; - float rho2d = FilterInvSquare * (d.x * d.x + d.y * d.y); - - // compute intersection and depth - float rho = min(rho3d, rho2d); - float depth = (rho3d <= rho2d) ? (s.x * Tw.x + s.y * Tw.y) + Tw.z : Tw.z; - if (depth < near_n) continue; - float4 nor_o = collected_normal_opacity[j]; - float normal[3] = {nor_o.x, nor_o.y, nor_o.z}; - float opa = nor_o.w; - - float power = -0.5f * rho; - if (power > 0.0f) - continue; - - // Eq. (2) from 3D Gaussian splatting paper. - // Obtain alpha by multiplying with Gaussian opacity - // and its exponential falloff from mean. - // Avoid numerical instabilities (see paper appendix). - float alpha = min(0.99f, opa * exp(power)); - if (alpha < 1.0f / 255.0f) - continue; - float test_T = T * (1 - alpha); - if (test_T < 0.0001f) - { - done = true; - continue; - } - - float w = alpha * T; - */ void main () { const float FilterInvSquare = 2.0; diff --git a/src/splatmesh/SplatMaterial3D.js b/src/splatmesh/SplatMaterial3D.js index 7249fa00..ff4bdea5 100644 --- a/src/splatmesh/SplatMaterial3D.js +++ b/src/splatmesh/SplatMaterial3D.js @@ -203,7 +203,7 @@ export class SplatMaterial3D { vertexShaderSource += ` vec2 ndcOffset = vec2(vPosition.x * basisVector1 + vPosition.y * basisVector2) * - basisViewport * 2.0 * inverseFocalAdjustment; + basisViewport * 2.0 * inverseFocalAdjustment; vec4 quadPos = vec4(ndcCenter.xy + ndcOffset, ndcCenter.z, 1.0); gl_Position = quadPos; diff --git a/src/splatmesh/SplatMesh.js b/src/splatmesh/SplatMesh.js index 1ab21805..f44d3ac8 100644 --- a/src/splatmesh/SplatMesh.js +++ b/src/splatmesh/SplatMesh.js @@ -46,7 +46,7 @@ const MAX_TEXTURE_TEXELS = 16777216; */ export class SplatMesh extends THREE.Mesh { - constructor(splatRenderMode = SplatRenderMode.ThreeD, dynamicMode = true, enableOptionalEffects = false, + constructor(splatRenderMode = SplatRenderMode.ThreeD, dynamicMode = false, enableOptionalEffects = false, halfPrecisionCovariancesOnGPU = false, devicePixelRatio = 1, enableDistancesComputationOnGPU = true, integerBasedDistancesComputation = false, antialiased = false, maxScreenSpaceSplatSize = 1024, logLevel = LogLevel.None, sphericalHarmonicsDegree = 0) { @@ -164,7 +164,7 @@ export class SplatMesh extends THREE.Mesh { * } * @return {Array} */ - static buildScenes(splatBuffers, sceneOptions) { + static buildScenes(parentObject, splatBuffers, sceneOptions) { const scenes = []; scenes.length = splatBuffers.length; for (let i = 0; i < splatBuffers.length; i++) { @@ -176,8 +176,10 @@ export class SplatMesh extends THREE.Mesh { const position = new THREE.Vector3().fromArray(positionArray); const rotation = new THREE.Quaternion().fromArray(rotationArray); const scale = new THREE.Vector3().fromArray(scaleArray); - scenes[i] = SplatMesh.createScene(splatBuffer, position, rotation, scale, - options.splatAlphaRemovalThreshold || 1, options.opacity, options.visible); + const scene = SplatMesh.createScene(splatBuffer, position, rotation, scale, + options.splatAlphaRemovalThreshold || 1, options.opacity, options.visible); + parentObject.add(scene); + scenes[i] = scene; } return scenes; } @@ -303,7 +305,7 @@ export class SplatMesh extends THREE.Mesh { const maxSplatCount = SplatMesh.getTotalMaxSplatCountForSplatBuffers(splatBuffers); - const newScenes = SplatMesh.buildScenes(splatBuffers, sceneOptions); + const newScenes = SplatMesh.buildScenes(this, splatBuffers, sceneOptions); if (keepSceneTransforms) { for (let i = 0; i < this.scenes.length && i < newScenes.length; i++) { const newScene = newScenes[i]; @@ -611,15 +613,18 @@ export class SplatMesh extends THREE.Mesh { * @param {boolean} sinceLastBuildOnly Specify whether or not to only update for splats that have been added since the last build. */ refreshDataTexturesFromSplatBuffers(sinceLastBuildOnly) { + const splatCount = this.getSplatCount(); + const fromSplat = this.lastBuildSplatCount; + const toSplat = splatCount - 1; + if (!sinceLastBuildOnly) { this.setupDataTextures(); + this.updateBaseDataFromSplatBuffers(); } else { - const splatCount = this.getSplatCount(); - const fromSplat = this.lastBuildSplatCount; - const toSplat = splatCount - 1; this.updateBaseDataFromSplatBuffers(fromSplat, toSplat); - this.updateDataTexturesFromBaseData(fromSplat, toSplat); } + + this.updateDataTexturesFromBaseData(fromSplat, toSplat); this.updateVisibleRegion(sinceLastBuildOnly); } @@ -672,9 +677,6 @@ export class SplatMesh extends THREE.Mesh { const shComponentCount = getSphericalHarmonicsComponentCountForDegree(this.minSphericalHarmonicsDegree); const shData = this.minSphericalHarmonicsDegree ? new SphericalHarmonicsArrayType(maxSplatCount * shComponentCount) : undefined; - this.fillSplatDataArrays(covariances, scales, rotations, centers, colors, shData, undefined, - covarianceCompressionLevel, scaleRotationCompressionLevel, shCompressionLevel); - // set up centers/colors data texture const centersColsTexSize = computeDataTextureSize(CENTER_COLORS_ELEMENTS_PER_TEXEL, 4); const paddedCentersCols = new Uint32Array(centersColsTexSize.x * centersColsTexSize.y * CENTER_COLORS_ELEMENTS_PER_TEXEL); @@ -860,27 +862,33 @@ export class SplatMesh extends THREE.Mesh { this.material.uniforms.sphericalHarmonicsTextureSize.value.copy(shTexSize); this.material.uniforms.sphericalHarmonics8BitMode.value = shCompressionLevel === 2 ? 1 : 0; + for (let s = 0; s < this.scenes.length; s++) { + const splatBuffer = this.scenes[s].splatBuffer; + this.material.uniforms.sphericalHarmonics8BitCompressionRangeMin.value[s] = + splatBuffer.minSphericalHarmonicsCoeff; + this.material.uniforms.sphericalHarmonics8BitCompressionRangeMax.value[s] = + splatBuffer.maxSphericalHarmonicsCoeff; + } this.material.uniformsNeedUpdate = true; } - if (this.dynamicMode || this.enableOptionalEffects) { - const sceneIndexesTexSize = computeDataTextureSize(SCENE_INDEXES_ELEMENTS_PER_TEXEL, 4); - const paddedTransformIndexes = new Uint32Array(sceneIndexesTexSize.x * - sceneIndexesTexSize.y * SCENE_INDEXES_ELEMENTS_PER_TEXEL); - for (let c = 0; c < splatCount; c++) paddedTransformIndexes[c] = this.globalSplatIndexToSceneIndexMap[c]; - const sceneIndexesTexture = new THREE.DataTexture(paddedTransformIndexes, sceneIndexesTexSize.x, sceneIndexesTexSize.y, - THREE.RedIntegerFormat, THREE.UnsignedIntType); - sceneIndexesTexture.internalFormat = 'R32UI'; - sceneIndexesTexture.needsUpdate = true; - this.material.uniforms.sceneIndexesTexture.value = sceneIndexesTexture; - this.material.uniforms.sceneIndexesTextureSize.value.copy(sceneIndexesTexSize); - this.material.uniformsNeedUpdate = true; - this.splatDataTextures['sceneIndexes'] = { - 'data': paddedTransformIndexes, - 'texture': sceneIndexesTexture, - 'size': sceneIndexesTexSize - }; - } + const sceneIndexesTexSize = computeDataTextureSize(SCENE_INDEXES_ELEMENTS_PER_TEXEL, 4); + const paddedTransformIndexes = new Uint32Array(sceneIndexesTexSize.x * + sceneIndexesTexSize.y * SCENE_INDEXES_ELEMENTS_PER_TEXEL); + for (let c = 0; c < splatCount; c++) paddedTransformIndexes[c] = this.globalSplatIndexToSceneIndexMap[c]; + const sceneIndexesTexture = new THREE.DataTexture(paddedTransformIndexes, sceneIndexesTexSize.x, sceneIndexesTexSize.y, + THREE.RedIntegerFormat, THREE.UnsignedIntType); + sceneIndexesTexture.internalFormat = 'R32UI'; + sceneIndexesTexture.needsUpdate = true; + this.material.uniforms.sceneIndexesTexture.value = sceneIndexesTexture; + this.material.uniforms.sceneIndexesTextureSize.value.copy(sceneIndexesTexSize); + this.material.uniformsNeedUpdate = true; + this.splatDataTextures['sceneIndexes'] = { + 'data': paddedTransformIndexes, + 'texture': sceneIndexesTexture, + 'size': sceneIndexesTexSize + }; + this.material.uniforms.sceneCount.value = this.scenes.length; } updateBaseDataFromSplatBuffers(fromSplat, toSplat) { @@ -1028,21 +1036,18 @@ export class SplatMesh extends THREE.Mesh { } // update scene index & transform data - if (this.dynamicMode) { - const sceneIndexesTexDesc = this.splatDataTextures['sceneIndexes']; - const paddedTransformIndexes = sceneIndexesTexDesc.data; - for (let c = this.lastBuildSplatCount; c <= toSplat; c++) { - paddedTransformIndexes[c] = this.globalSplatIndexToSceneIndexMap[c]; - } - - const sceneIndexesTexture = sceneIndexesTexDesc.texture; - const sceneIndexesTextureProps = this.renderer ? this.renderer.properties.get(sceneIndexesTexture) : null; - if (!sceneIndexesTextureProps || !sceneIndexesTextureProps.__webglTexture) { - sceneIndexesTexture.needsUpdate = true; - } else { - this.updateDataTexture(paddedTransformIndexes, sceneIndexesTexDesc.texture, sceneIndexesTexDesc.size, - sceneIndexesTextureProps, 1, 1, 1, this.lastBuildSplatCount, toSplat); - } + const sceneIndexesTexDesc = this.splatDataTextures['sceneIndexes']; + const paddedSceneIndexes = sceneIndexesTexDesc.data; + for (let c = this.lastBuildSplatCount; c <= toSplat; c++) { + paddedSceneIndexes[c] = this.globalSplatIndexToSceneIndexMap[c]; + } + const sceneIndexesTexture = sceneIndexesTexDesc.texture; + const sceneIndexesTextureProps = this.renderer ? this.renderer.properties.get(sceneIndexesTexture) : null; + if (!sceneIndexesTextureProps || !sceneIndexesTextureProps.__webglTexture) { + sceneIndexesTexture.needsUpdate = true; + } else { + this.updateDataTexture(paddedSceneIndexes, sceneIndexesTexDesc.texture, sceneIndexesTexDesc.size, + sceneIndexesTextureProps, 1, 1, 1, this.lastBuildSplatCount, toSplat); } } @@ -1174,7 +1179,7 @@ export class SplatMesh extends THREE.Mesh { const startSplatFormMaxDistanceCalc = sinceLastBuildOnly ? this.lastBuildSplatCount : 0; for (let i = startSplatFormMaxDistanceCalc; i < splatCount; i++) { - this.getSplatCenter(i, tempCenter, false); + this.getSplatCenter(i, tempCenter, true); const distFromCSceneCenter = tempCenter.sub(this.calculatedSceneCenter).length(); if (distFromCSceneCenter > this.maxSplatDistanceFromSceneCenter) this.maxSplatDistanceFromSceneCenter = distFromCSceneCenter; } @@ -1229,7 +1234,7 @@ export class SplatMesh extends THREE.Mesh { updateTransforms() { for (let i = 0; i < this.scenes.length; i++) { const scene = this.getScene(i); - scene.updateTransform(); + scene.updateTransform(this.dynamicMode); } } @@ -1848,6 +1853,7 @@ export class SplatMesh extends THREE.Mesh { } else { scaleOverride.z = 1; } + const tempTransform = new THREE.Matrix4(); for (let i = 0; i < this.scenes.length; i++) { if (applySceneTransform === undefined || applySceneTransform === null) { @@ -1856,7 +1862,11 @@ export class SplatMesh extends THREE.Mesh { const scene = this.getScene(i); const splatBuffer = scene.splatBuffer; - const sceneTransform = applySceneTransform ? scene.transform : null; + let sceneTransform; + if (applySceneTransform) { + this.getSceneTransform(i, tempTransform); + sceneTransform = tempTransform; + } if (covariances) { splatBuffer.fillSplatCovarianceArray(covariances, sceneTransform, srcStart, srcEnd, destStart, covarianceCompressionLevel); } @@ -1994,7 +2004,7 @@ export class SplatMesh extends THREE.Mesh { */ getSceneTransform(sceneIndex, outTransform) { const scene = this.getScene(sceneIndex); - scene.updateTransform(); + scene.updateTransform(this.dynamicMode); outTransform.copy(scene.transform); } @@ -2010,6 +2020,10 @@ export class SplatMesh extends THREE.Mesh { return this.scenes[sceneIndex]; } + getSceneCount() { + return this.scenes.length; + } + getSplatBufferForSplat(globalIndex) { return this.getScene(this.globalSplatIndexToSceneIndexMap[globalIndex]).splatBuffer; } diff --git a/src/splatmesh/SplatScene.js b/src/splatmesh/SplatScene.js index f6053386..a5175ea6 100644 --- a/src/splatmesh/SplatScene.js +++ b/src/splatmesh/SplatScene.js @@ -3,19 +3,19 @@ import * as THREE from 'three'; /** * SplatScene: Descriptor for a single splat scene managed by an instance of SplatMesh. */ -export class SplatScene { +export class SplatScene extends THREE.Object3D { constructor(splatBuffer, position = new THREE.Vector3(), quaternion = new THREE.Quaternion(), scale = new THREE.Vector3(1, 1, 1), minimumAlpha = 1, opacity = 1.0, visible = true) { + super(); this.splatBuffer = splatBuffer; - this.position = position.clone(); - this.quaternion = quaternion.clone(); - this.scale = scale.clone(); + this.position.copy(position); + this.quaternion.copy(quaternion); + this.scale.copy(scale); this.transform = new THREE.Matrix4(); this.minimumAlpha = minimumAlpha; this.opacity = opacity; this.visible = visible; - this.updateTransform(); } copyTransformData(otherScene) { @@ -25,7 +25,13 @@ export class SplatScene { this.transform.copy(otherScene.transform); } - updateTransform() { - this.transform.compose(this.position, this.quaternion, this.scale); + updateTransform(dynamicMode) { + if (dynamicMode) { + if (this.matrixWorldAutoUpdate) this.updateWorldMatrix(true, false); + this.transform.copy(this.matrixWorld); + } else { + if (this.matrixAutoUpdate) this.updateMatrix(); + this.transform.copy(this.matrix); + } } } diff --git a/src/ui/InfoPanel.js b/src/ui/InfoPanel.js index 4c367c54..0b01a08e 100644 --- a/src/ui/InfoPanel.js +++ b/src/ui/InfoPanel.js @@ -146,7 +146,7 @@ export class InfoPanel { }; setContainer(container) { - if (this.container) { + if (this.container && this.infoPanelContainer.parentElement === this.container) { this.container.removeChild(this.infoPanelContainer); } if (container) { diff --git a/src/ui/LoadingProgressBar.js b/src/ui/LoadingProgressBar.js index c16e0519..242cf109 100644 --- a/src/ui/LoadingProgressBar.js +++ b/src/ui/LoadingProgressBar.js @@ -89,7 +89,7 @@ export class LoadingProgressBar { } setContainer(container) { - if (this.container) { + if (this.container && this.progressBarContainerOuter.parentElement === this.container) { this.container.removeChild(this.progressBarContainerOuter); } if (container) { diff --git a/src/ui/LoadingSpinner.js b/src/ui/LoadingSpinner.js index f5100cac..ba8cb336 100644 --- a/src/ui/LoadingSpinner.js +++ b/src/ui/LoadingSpinner.js @@ -212,7 +212,7 @@ export class LoadingSpinner { } setContainer(container) { - if (this.container) { + if (this.container && this.spinnerContainerOuter.parentElement === this.container) { this.container.removeChild(this.spinnerContainerOuter); } if (container) { diff --git a/src/worker/SortWorker.js b/src/worker/SortWorker.js index 2d18aaab..9b73a0f1 100644 --- a/src/worker/SortWorker.js +++ b/src/worker/SortWorker.js @@ -212,13 +212,13 @@ export function createSortWorker(splatCount, useSharedMemory, enableSIMDInSort, let iOSSemVer = isIOS() ? getIOSSemever() : null; if (!enableSIMDInSort && !useSharedMemory) { sourceWasm = SorterWasmNoSIMD; - if (iOSSemVer && iOSSemVer.major < 16) { + if (iOSSemVer && iOSSemVer.major < 16 && iOSSemVer.minor < 4) { sourceWasm = SorterWasmNoSIMDNonShared; } } else if (!enableSIMDInSort) { sourceWasm = SorterWasmNoSIMD; } else if (!useSharedMemory) { - if (iOSSemVer && iOSSemVer.major < 16) { + if (iOSSemVer && iOSSemVer.major < 16 && iOSSemVer.minor < 4) { sourceWasm = SorterWasmNonShared; } }