From fe131e6dd4491a6f5e31de20ada8cf07ae43f5de Mon Sep 17 00:00:00 2001 From: Junya Kuwada Date: Fri, 20 Oct 2023 19:56:21 +0900 Subject: [PATCH] Add pixelRatio and xrPixelRatio parameters --- README.md | 2 ++ dist/aframe-gaussian-splatting-component.min.js | 4 ++-- dist/aframe-gaussian-splatting-component.min.js.map | 2 +- index.js | 12 +++++++++--- package.json | 2 +- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e9b2c62..8e21119 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ Needs few seconds to load splats. | -------- | ----------- | ------------- | | src | url of splat or ply file | train.splat | | cutoutEntity | selector to a box primitive that uses scale and position to define the bounds of splat points to render | | +| pixelRatio | Pixel ratio for rendering. Reducing the value decreases the resolution and improves performance. If a negative value is set, the device's native value will be applied. | 1.0 | +| xrPixelRatio | Same as pixelRatio. Applied to XR devices. | 0.5 | ### Example custom scan to gaussian splat workflow * Use a service such as https://lumalabs.ai/ to process a scan into splat (an alternative is https://poly.cam/) diff --git a/dist/aframe-gaussian-splatting-component.min.js b/dist/aframe-gaussian-splatting-component.min.js index e9df176..71fc777 100644 --- a/dist/aframe-gaussian-splatting-component.min.js +++ b/dist/aframe-gaussian-splatting-component.min.js @@ -1,4 +1,4 @@ -AFRAME.registerComponent("gaussian_splatting",{schema:{src:{type:"string",default:"train.splat"}},init:function(){this.el.sceneEl.renderer.setPixelRatio(1),this.el.sceneEl.renderer.xr.setFramebufferScaleFactor(.5),this.loadData(this.el.sceneEl.camera.el.components.camera.camera,this.el.object3D,this.el.sceneEl.renderer,this.data.src)},initGL:async function(numVertexes){console.log("initGL",numVertexes),this.object.frustumCulled=!1;var gl=this.renderer.getContext(),gl=gl.getParameter(gl.MAX_TEXTURE_SIZE),numVertexes=(this.maxVertexes=gl*gl,numVertexes>this.maxVertexes&&(console.log("numVertexes limited to ",this.maxVertexes,numVertexes),numVertexes=this.maxVertexes),this.bufferTextureWidth=gl,this.bufferTextureHeight=Math.floor((numVertexes-1)/gl)+1,this.centerAndScaleData=new Float32Array(this.bufferTextureWidth*this.bufferTextureHeight*4),this.covAndColorData=new Uint32Array(this.bufferTextureWidth*this.bufferTextureHeight*4),this.centerAndScaleTexture=new THREE.DataTexture(this.centerAndScaleData,this.bufferTextureWidth,this.bufferTextureHeight,THREE.RGBA,THREE.FloatType),this.centerAndScaleTexture.needsUpdate=!0,this.covAndColorTexture=new THREE.DataTexture(this.covAndColorData,this.bufferTextureWidth,this.bufferTextureHeight,THREE.RGBAIntegerFormat,THREE.UnsignedIntType),this.covAndColorTexture.internalFormat="RGBA32UI",this.covAndColorTexture.needsUpdate=!0,new Uint32Array(this.bufferTextureWidth*this.bufferTextureHeight)),gl=new THREE.InstancedBufferAttribute(numVertexes,1,!1),numVertexes=(gl.setUsage(THREE.DynamicDrawUsage),new THREE.BufferGeometry),positionsArray=new Float32Array(18),positionsArray=new THREE.BufferAttribute(positionsArray,3),positionsArray=(numVertexes.setAttribute("position",positionsArray),positionsArray.setXYZ(2,-2,2,0),positionsArray.setXYZ(1,2,2,0),positionsArray.setXYZ(0,-2,-2,0),positionsArray.setXYZ(5,-2,-2,0),positionsArray.setXYZ(4,2,2,0),positionsArray.setXYZ(3,2,-2,0),positionsArray.needsUpdate=!0,(new THREE.InstancedBufferGeometry).copy(numVertexes));positionsArray.setAttribute("splatIndex",gl),positionsArray.instanceCount=1;const material=new THREE.ShaderMaterial({uniforms:{viewport:{value:new Float32Array([1980,1080])},focal:{value:1e3},centerAndScaleTexture:{value:this.centerAndScaleTexture},covAndColorTexture:{value:this.covAndColorTexture},gsProjectionMatrix:{value:this.getProjectionMatrix()},gsModelViewMatrix:{value:this.getModelViewMatrix()}},vertexShader:` +AFRAME.registerComponent("gaussian_splatting",{schema:{src:{type:"string",default:"train.splat"},cutoutEntity:{type:"selector"},pixelRatio:{type:"number",default:1},xrPixelRatio:{type:"number",default:.5}},init:function(){0this.maxVertexes&&(console.log("numVertexes limited to ",this.maxVertexes,numVertexes),numVertexes=this.maxVertexes),this.bufferTextureWidth=gl,this.bufferTextureHeight=Math.floor((numVertexes-1)/gl)+1,this.centerAndScaleData=new Float32Array(this.bufferTextureWidth*this.bufferTextureHeight*4),this.covAndColorData=new Uint32Array(this.bufferTextureWidth*this.bufferTextureHeight*4),this.centerAndScaleTexture=new THREE.DataTexture(this.centerAndScaleData,this.bufferTextureWidth,this.bufferTextureHeight,THREE.RGBA,THREE.FloatType),this.centerAndScaleTexture.needsUpdate=!0,this.covAndColorTexture=new THREE.DataTexture(this.covAndColorData,this.bufferTextureWidth,this.bufferTextureHeight,THREE.RGBAIntegerFormat,THREE.UnsignedIntType),this.covAndColorTexture.internalFormat="RGBA32UI",this.covAndColorTexture.needsUpdate=!0,new Uint32Array(this.bufferTextureWidth*this.bufferTextureHeight)),gl=new THREE.InstancedBufferAttribute(numVertexes,1,!1),numVertexes=(gl.setUsage(THREE.DynamicDrawUsage),new THREE.BufferGeometry),positionsArray=new Float32Array(18),positionsArray=new THREE.BufferAttribute(positionsArray,3),positionsArray=(numVertexes.setAttribute("position",positionsArray),positionsArray.setXYZ(2,-2,2,0),positionsArray.setXYZ(1,2,2,0),positionsArray.setXYZ(0,-2,-2,0),positionsArray.setXYZ(5,-2,-2,0),positionsArray.setXYZ(4,2,2,0),positionsArray.setXYZ(3,2,-2,0),positionsArray.needsUpdate=!0,(new THREE.InstancedBufferGeometry).copy(numVertexes));positionsArray.setAttribute("splatIndex",gl),positionsArray.instanceCount=1;const material=new THREE.ShaderMaterial({uniforms:{viewport:{value:new Float32Array([1980,1080])},focal:{value:1e3},centerAndScaleTexture:{value:this.centerAndScaleTexture},covAndColorTexture:{value:this.covAndColorTexture},gsProjectionMatrix:{value:this.getProjectionMatrix()},gsModelViewMatrix:{value:this.getModelViewMatrix()}},vertexShader:` precision highp sampler2D; precision highp usampler2D; @@ -96,4 +96,4 @@ AFRAME.registerComponent("gaussian_splatting",{schema:{src:{type:"string",defaul float B = exp(A) * vColor.a; gl_FragColor = vec4(vColor.rgb, B); } - `,blending:THREE.CustomBlending,blendSrcAlpha:THREE.OneFactor,depthTest:!0,depthWrite:!1,transparent:!0}),mesh=(material.onBeforeRender=(renderer,scene,camera,geometry,object,group)=>{var projectionMatrix=this.getProjectionMatrix(camera),camera=(mesh.material.uniforms.gsProjectionMatrix.value=projectionMatrix,mesh.material.uniforms.gsModelViewMatrix.value=this.getModelViewMatrix(camera),new THREE.Vector4),renderer=(renderer.getCurrentViewport(camera),camera.w/2*Math.abs(projectionMatrix.elements[5]));material.uniforms.viewport.value[0]=camera.z,material.uniforms.viewport.value[1]=camera.w,material.uniforms.focal.value=renderer},new THREE.Mesh(positionsArray,material));for(mesh.frustumCulled=!1,this.object.add(mesh),this.worker.onmessage=e=>{e=new Uint32Array(e.data.sortedIndexes);mesh.geometry.attributes.splatIndex.set(e),mesh.geometry.attributes.splatIndex.needsUpdate=!0,mesh.geometry.instanceCount=e.length,this.sortReady=!0};;){var centerAndScaleTextureProperties=this.renderer.properties.get(this.centerAndScaleTexture),covAndColorTextureProperties=this.renderer.properties.get(this.covAndColorTexture);if(centerAndScaleTextureProperties&¢erAndScaleTextureProperties.__webglTexture&&covAndColorTextureProperties&¢erAndScaleTextureProperties.__webglTexture)break;await new Promise(resolve=>setTimeout(resolve,10))}this.sortReady=!0},loadData:function(camera,object,renderer,src){this.camera=camera,this.object=object,this.renderer=renderer,this.loadedVertexCount=0,this.rowLength=32,this.worker=new Worker(URL.createObjectURL(new Blob(["(",this.createWorker.toString(),")(self)"],{type:"application/javascript"}))),this.worker.postMessage({method:"clear"}),fetch(src).then(async data=>{var reader=data.body.getReader();let glInitialized=!1,bytesDownloaded=0,bytesProcesses=0;var data=data.headers.get("Content-Length"),totalDownloadBytes=data?parseInt(data):void 0,chunks=(null!=totalDownloadBytes&&(data=Math.floor(totalDownloadBytes/this.rowLength),await this.initGL(data),glInitialized=!0),[]),start=Date.now();let lastReportedProgress=0;for(var isPly=src.endsWith(".ply");;)try{var mbps,percent,{value,done}=await reader.read();if(done){console.log("Completed download.");break}bytesDownloaded+=value.length,null!=totalDownloadBytes?(mbps=bytesDownloaded/1024/1024/((Date.now()-start)/1e3),1<(percent=bytesDownloaded/totalDownloadBytes*100)-lastReportedProgress&&(console.log("download progress:",percent.toFixed(2)+"%",mbps.toFixed(2)+" Mbps"),lastReportedProgress=percent)):console.log("download progress:",bytesDownloaded,", unknown total"),chunks.push(value);var bytesRemains=bytesDownloaded-bytesProcesses;if(!isPly&&null!=totalDownloadBytes&&bytesRemains>this.rowLength){var extra_data,vertexCount=Math.floor(bytesRemains/this.rowLength),concatenatedChunksbuffer=new Uint8Array(bytesRemains);let offset=0;for(const chunk of chunks)concatenatedChunksbuffer.set(chunk,offset),offset+=chunk.length;chunks.length=0,bytesRemains>vertexCount*this.rowLength&&((extra_data=new Uint8Array(bytesRemains-vertexCount*this.rowLength)).set(concatenatedChunksbuffer.subarray(bytesRemains-extra_data.length,bytesRemains),0),chunks.push(extra_data));var buffer=new Uint8Array(vertexCount*this.rowLength);buffer.set(concatenatedChunksbuffer.subarray(0,buffer.byteLength),0),this.pushDataBuffer(buffer.buffer,vertexCount),bytesProcesses+=vertexCount*this.rowLength}}catch(error){console.error(error);break}if(0acc+chunk.length,0)),offset=0;for(const chunk of chunks)concatenatedChunks.set(chunk,offset),offset+=chunk.length;isPly&&(concatenatedChunks=new Uint8Array(this.processPlyBuffer(concatenatedChunks.buffer)));let numVertexes=Math.floor(concatenatedChunks.byteLength/this.rowLength);glInitialized||(await this.initGL(numVertexes),glInitialized=!0),this.pushDataBuffer(concatenatedChunks.buffer,numVertexes)}})},pushDataBuffer:function(buffer,vertexCount){if(this.loadedVertexCount+vertexCount>this.maxVertexes&&(console.log("vertexCount limited to ",this.maxVertexes,vertexCount),vertexCount=this.maxVertexes-this.loadedVertexCount),!(vertexCount<=0)){var u_buffer=new Uint8Array(buffer),f_buffer=new Float32Array(buffer),matrices=new Float32Array(16*vertexCount),covAndColorData_uint8=new Uint8Array(this.covAndColorData.buffer),covAndColorData_int16=new Int16Array(this.covAndColorData.buffer);for(let i=0;imax_value&&(max_value=Math.abs(mtx.elements[cov_indexes[j]]));let destOffset=4*this.loadedVertexCount+4*i;this.centerAndScaleData[destOffset+0]=center.x,this.centerAndScaleData[destOffset+1]=center.y,this.centerAndScaleData[destOffset+2]=center.z,this.centerAndScaleData[destOffset+3]=max_value/32767,destOffset=8*this.loadedVertexCount+4*i*2;for(let j=0;j-1e-4*depth&&(depthList[validCount]=depth,validIndexList[validCount]=i,validCount++,depth>maxDepth&&(maxDepth=depth),depth{if("clear"==e.data.method&&(matrices=void 0),"push"==e.data.method&&(new_matrices=new Float32Array(e.data.matrices),matrices=void 0===matrices?new_matrices:((resized=new Float32Array(matrices.length+new_matrices.length)).set(matrices),resized.set(new_matrices,matrices.length),resized)),"sort"==e.data.method)if(void 0===matrices){var new_matrices=new Uint32Array(1);self.postMessage({sortedIndexes:new_matrices},[new_matrices.buffer])}else{new_matrices=new Float32Array(e.data.view);const sortedIndexes=sortSplats(matrices,new_matrices);self.postMessage({sortedIndexes:sortedIndexes},[sortedIndexes.buffer])}}},processPlyBuffer:function(inputBuffer){var ubuf=new Uint8Array(inputBuffer),ubuf=(new TextDecoder).decode(ubuf.slice(0,10240)),header_end_index=ubuf.indexOf("end_header\n");if(header_end_index<0)throw new Error("Unable to read .ply file header");var vertexCount=parseInt(/element vertex (\d+)\n/.exec(ubuf)[1]);console.log("Vertex Count",vertexCount);let row_offset=0,offsets={},types={};var prop,TYPE_MAP={double:"getFloat64",int:"getInt32",uint:"getUint32",float:"getFloat32",short:"getInt16",ushort:"getUint16",uchar:"getUint8"};for(prop of ubuf.slice(0,header_end_index).split("\n").filter(k=>k.startsWith("property "))){var[,type,name]=prop.split(" "),type=TYPE_MAP[type]||"getInt8";types[name]=type,offsets[name]=row_offset,row_offset+=parseInt(type.replace(/[^\d]/g,""))/8}console.log("Bytes per row",row_offset,types,offsets);let dataView=new DataView(inputBuffer,header_end_index+"end_header\n".length),row=0;var attrs=new Proxy({},{get(target,prop){if(types[prop])return dataView[types[prop]](row*row_offset+offsets[prop],!0);throw new Error(prop+" not found")}});console.time("calculate importance");let sizeList=new Float32Array(vertexCount);var size,opacity,sizeIndex=new Uint32Array(vertexCount);for(row=0;rowsizeList[a]-sizeList[b]),console.timeEnd("sort");var buffer=new ArrayBuffer(32*vertexCount);console.time("build buffer");for(let j=0;j{var projectionMatrix=this.getProjectionMatrix(camera),camera=(mesh.material.uniforms.gsProjectionMatrix.value=projectionMatrix,mesh.material.uniforms.gsModelViewMatrix.value=this.getModelViewMatrix(camera),new THREE.Vector4),renderer=(renderer.getCurrentViewport(camera),camera.w/2*Math.abs(projectionMatrix.elements[5]));material.uniforms.viewport.value[0]=camera.z,material.uniforms.viewport.value[1]=camera.w,material.uniforms.focal.value=renderer},new THREE.Mesh(positionsArray,material));for(mesh.frustumCulled=!1,this.object.add(mesh),this.worker.onmessage=e=>{e=new Uint32Array(e.data.sortedIndexes);mesh.geometry.attributes.splatIndex.set(e),mesh.geometry.attributes.splatIndex.needsUpdate=!0,mesh.geometry.instanceCount=e.length,this.sortReady=!0};;){var centerAndScaleTextureProperties=this.renderer.properties.get(this.centerAndScaleTexture),covAndColorTextureProperties=this.renderer.properties.get(this.covAndColorTexture);if(centerAndScaleTextureProperties&¢erAndScaleTextureProperties.__webglTexture&&covAndColorTextureProperties&¢erAndScaleTextureProperties.__webglTexture)break;await new Promise(resolve=>setTimeout(resolve,10))}this.sortReady=!0},loadData:function(camera,object,renderer,src){this.camera=camera,this.object=object,this.renderer=renderer,this.loadedVertexCount=0,this.rowLength=32,this.worker=new Worker(URL.createObjectURL(new Blob(["(",this.createWorker.toString(),")(self)"],{type:"application/javascript"}))),this.worker.postMessage({method:"clear"}),fetch(src).then(async data=>{var reader=data.body.getReader();let glInitialized=!1,bytesDownloaded=0,bytesProcesses=0;var data=data.headers.get("Content-Length"),totalDownloadBytes=data?parseInt(data):void 0,chunks=(null!=totalDownloadBytes&&(data=Math.floor(totalDownloadBytes/this.rowLength),await this.initGL(data),glInitialized=!0),[]),start=Date.now();let lastReportedProgress=0;for(var isPly=src.endsWith(".ply");;)try{var mbps,percent,{value,done}=await reader.read();if(done){console.log("Completed download.");break}bytesDownloaded+=value.length,null!=totalDownloadBytes?(mbps=bytesDownloaded/1024/1024/((Date.now()-start)/1e3),1<(percent=bytesDownloaded/totalDownloadBytes*100)-lastReportedProgress&&(console.log("download progress:",percent.toFixed(2)+"%",mbps.toFixed(2)+" Mbps"),lastReportedProgress=percent)):console.log("download progress:",bytesDownloaded,", unknown total"),chunks.push(value);var bytesRemains=bytesDownloaded-bytesProcesses;if(!isPly&&null!=totalDownloadBytes&&bytesRemains>this.rowLength){var extra_data,vertexCount=Math.floor(bytesRemains/this.rowLength),concatenatedChunksbuffer=new Uint8Array(bytesRemains);let offset=0;for(const chunk of chunks)concatenatedChunksbuffer.set(chunk,offset),offset+=chunk.length;chunks.length=0,bytesRemains>vertexCount*this.rowLength&&((extra_data=new Uint8Array(bytesRemains-vertexCount*this.rowLength)).set(concatenatedChunksbuffer.subarray(bytesRemains-extra_data.length,bytesRemains),0),chunks.push(extra_data));var buffer=new Uint8Array(vertexCount*this.rowLength);buffer.set(concatenatedChunksbuffer.subarray(0,buffer.byteLength),0),this.pushDataBuffer(buffer.buffer,vertexCount),bytesProcesses+=vertexCount*this.rowLength}}catch(error){console.error(error);break}if(0acc+chunk.length,0)),offset=0;for(const chunk of chunks)concatenatedChunks.set(chunk,offset),offset+=chunk.length;isPly&&(concatenatedChunks=new Uint8Array(this.processPlyBuffer(concatenatedChunks.buffer)));let numVertexes=Math.floor(concatenatedChunks.byteLength/this.rowLength);glInitialized||(await this.initGL(numVertexes),glInitialized=!0),this.pushDataBuffer(concatenatedChunks.buffer,numVertexes)}})},pushDataBuffer:function(buffer,vertexCount){if(this.loadedVertexCount+vertexCount>this.maxVertexes&&(console.log("vertexCount limited to ",this.maxVertexes,vertexCount),vertexCount=this.maxVertexes-this.loadedVertexCount),!(vertexCount<=0)){var u_buffer=new Uint8Array(buffer),f_buffer=new Float32Array(buffer),matrices=new Float32Array(16*vertexCount),covAndColorData_uint8=new Uint8Array(this.covAndColorData.buffer),covAndColorData_int16=new Int16Array(this.covAndColorData.buffer);for(let i=0;imax_value&&(max_value=Math.abs(mtx.elements[cov_indexes[j]]));let destOffset=4*this.loadedVertexCount+4*i;this.centerAndScaleData[destOffset+0]=center.x,this.centerAndScaleData[destOffset+1]=center.y,this.centerAndScaleData[destOffset+2]=center.z,this.centerAndScaleData[destOffset+3]=max_value/32767,destOffset=8*this.loadedVertexCount+4*i*2;for(let j=0;j-1e-4*depth&&cutoutArea&&(depthList[validCount]=depth,validIndexList[validCount]=i,validCount++,depth>maxDepth&&(maxDepth=depth),depth{if("clear"==e.data.method&&(matrices=void 0),"push"==e.data.method&&(new_matrices=new Float32Array(e.data.matrices),matrices=void 0===matrices?new_matrices:((resized=new Float32Array(matrices.length+new_matrices.length)).set(matrices),resized.set(new_matrices,matrices.length),resized)),"sort"==e.data.method)if(void 0===matrices){var new_matrices=new Uint32Array(1);self.postMessage({sortedIndexes:new_matrices},[new_matrices.buffer])}else{new_matrices=new Float32Array(e.data.view),e=void 0!==e.data.cutout?new Float32Array(e.data.cutout):void 0;const sortedIndexes=sortSplats(matrices,new_matrices,e);self.postMessage({sortedIndexes:sortedIndexes},[sortedIndexes.buffer])}}},processPlyBuffer:function(inputBuffer){var ubuf=new Uint8Array(inputBuffer),ubuf=(new TextDecoder).decode(ubuf.slice(0,10240)),header_end_index=ubuf.indexOf("end_header\n");if(header_end_index<0)throw new Error("Unable to read .ply file header");var vertexCount=parseInt(/element vertex (\d+)\n/.exec(ubuf)[1]);console.log("Vertex Count",vertexCount);let row_offset=0,offsets={},types={};var prop,TYPE_MAP={double:"getFloat64",int:"getInt32",uint:"getUint32",float:"getFloat32",short:"getInt16",ushort:"getUint16",uchar:"getUint8"};for(prop of ubuf.slice(0,header_end_index).split("\n").filter(k=>k.startsWith("property "))){var[,type,name]=prop.split(" "),type=TYPE_MAP[type]||"getInt8";types[name]=type,offsets[name]=row_offset,row_offset+=parseInt(type.replace(/[^\d]/g,""))/8}console.log("Bytes per row",row_offset,types,offsets);let dataView=new DataView(inputBuffer,header_end_index+"end_header\n".length),row=0;var attrs=new Proxy({},{get(target,prop){if(types[prop])return dataView[types[prop]](row*row_offset+offsets[prop],!0);throw new Error(prop+" not found")}});console.time("calculate importance");let sizeList=new Float32Array(vertexCount);var size,opacity,sizeIndex=new Uint32Array(vertexCount);for(row=0;rowsizeList[a]-sizeList[b]),console.timeEnd("sort");var buffer=new ArrayBuffer(32*vertexCount);console.time("build buffer");for(let j=0;j 0){ + this.el.sceneEl.renderer.setPixelRatio(this.data.pixelRatio); + } + if(this.data.xrPixelRatio > 0){ + this.el.sceneEl.renderer.xr.setFramebufferScaleFactor(this.data.xrPixelRatio); + } this.loadData(this.el.sceneEl.camera.el.components.camera.camera, this.el.object3D, this.el.sceneEl.renderer, this.data.src); if (!!this.data.cutoutEntity) { this.cutout = this.data.cutoutEntity.object3D; diff --git a/package.json b/package.json index f8a8ed6..b9c8def 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "aframe-gaussian-splatting-component", - "version": "0.0.21", + "version": "0.0.22", "description": "This component is an A-Frame implementation of real-time rendering for '3D Gaussian Splatting for Real-Time Radiance Field Rendering'", "main": "index.js", "scripts": {