From 0956e7cd17d68c5e915c7dfc3a70a1c1a1f2c39a Mon Sep 17 00:00:00 2001 From: WilliamLiu-1997 <77861330+WilliamLiu-1997@users.noreply.github.com> Date: Fri, 3 Apr 2026 18:29:24 +1100 Subject: [PATCH] Add camera-relative splat rendering support for large world coordinates --- src/SparkRenderer.ts | 7 +- src/SplatAccumulator.ts | 65 ++++++++++++++--- src/SplatGenerator.ts | 2 + src/SplatMesh.ts | 135 +++++++++++++++++++++++++++++++++-- src/modifiers/depthColor.ts | 4 +- src/modifiers/normalColor.ts | 6 +- 6 files changed, 196 insertions(+), 23 deletions(-) diff --git a/src/SparkRenderer.ts b/src/SparkRenderer.ts index 2c51bbbe..a9a204f7 100644 --- a/src/SparkRenderer.ts +++ b/src/SparkRenderer.ts @@ -672,10 +672,9 @@ export class SparkRenderer extends THREE.Mesh { const geometry = this.geometry as SplatGeometry; geometry.instanceCount = spark.activeSplats; - const accumToWorld = new THREE.Matrix4(); - if (!this.display.extSplats) { - accumToWorld.makeTranslation(spark.display.viewOrigin); - } + const accumToWorld = new THREE.Matrix4().makeTranslation( + spark.display.viewOrigin, + ); const cameraToWorld = camera.matrixWorld.clone(); const worldToCamera = cameraToWorld.invert(); const accumToCamera = worldToCamera.multiply(accumToWorld); diff --git a/src/SplatAccumulator.ts b/src/SplatAccumulator.ts index 62413570..4b7ee69f 100644 --- a/src/SplatAccumulator.ts +++ b/src/SplatAccumulator.ts @@ -210,12 +210,28 @@ export class SplatAccumulator { prepareProgramMaterial( generator?: GsplatGenerator, covGenerator?: CovSplatGenerator, + useRelativeCenter = false, ) { const theGenerator = generator ?? covGenerator; if (!theGenerator) { throw new Error("Either generator or covGenerator must be provided"); } + // Invalidate cached program if the relative center mode changed + const cachedRelative = + SplatAccumulator.generatorRelativeCenter.get(theGenerator); + if (cachedRelative !== undefined && cachedRelative !== useRelativeCenter) { + SplatAccumulator.generatorProgram.delete(theGenerator); + } + SplatAccumulator.generatorRelativeCenter.set( + theGenerator, + useRelativeCenter, + ); + + const viewCenter = useRelativeCenter + ? SplatAccumulator.relativeCenterZero + : SplatAccumulator.viewCenterUniform; + let program = SplatAccumulator.generatorProgram.get(theGenerator); if (!program) { const graph = dynoBlock( @@ -232,29 +248,39 @@ export class SplatAccumulator { if (this.extSplats) { if (!this.covSplats) { if (generator) { - const output = outputExtendedSplat(generator.outputs.gsplat); + const split = splitGsplat(generator.outputs.gsplat).outputs; + const gsplat = combineGsplat({ + gsplat: generator.outputs.gsplat, + center: sub(split.center, viewCenter), + }); + const output = outputExtendedSplat(gsplat); roots.push(output); } else { throw new Error("Generator must be provided"); } } else { + let covsplat: DynoVal; if (covGenerator) { - const output = outputExtCovSplat(covGenerator.outputs.covsplat); - roots.push(output); + covsplat = covGenerator.outputs.covsplat; } else if (generator) { - const covsplat = gsplatToCovSplat(generator.outputs.gsplat); - const output = outputExtCovSplat(covsplat); - roots.push(output); + covsplat = gsplatToCovSplat(generator.outputs.gsplat); } else { throw new Error("Generator must be provided"); } + const split = splitCovSplat(covsplat).outputs; + const centeredCovSplat = combineCovSplat({ + covsplat, + center: sub(split.center, viewCenter), + }); + const output = outputExtCovSplat(centeredCovSplat); + roots.push(output); } } else { if (!this.covSplats) { if (generator) { const centerSubView = sub( splitGsplat(generator.outputs.gsplat).outputs.center, - SplatAccumulator.viewCenterUniform, + viewCenter, ); // Use expanded LoD opacity encoding const halfAlpha = mul( @@ -285,7 +311,7 @@ export class SplatAccumulator { } const centerSubView = sub( splitCovSplat(covsplat).outputs.center, - SplatAccumulator.viewCenterUniform, + viewCenter, ); const halfAlpha = mul( splitCovSplat(covsplat).outputs.opacity, @@ -309,7 +335,7 @@ export class SplatAccumulator { if (generator) { const outputDepth = outputSplatDepth( generator.outputs.gsplat, - SplatAccumulator.viewCenterUniform, + viewCenter, SplatAccumulator.viewDirUniform, SplatAccumulator.sortRadialUniform, ); @@ -318,7 +344,7 @@ export class SplatAccumulator { if (covGenerator) { const outputDepth = outputCovSplatDepth( covGenerator.outputs.covsplat, - SplatAccumulator.viewCenterUniform, + viewCenter, SplatAccumulator.viewDirUniform, SplatAccumulator.sortRadialUniform, ); @@ -358,6 +384,13 @@ export class SplatAccumulator { GsplatGenerator | CovSplatGenerator, DynoProgram >(); + static generatorRelativeCenter = new Map< + GsplatGenerator | CovSplatGenerator, + boolean + >(); + static relativeCenterZero = new DynoVec3({ + value: new THREE.Vector3(), + }); static fullScreenQuad = new FullScreenQuad( new THREE.RawShaderMaterial({ visible: false }), ); @@ -368,12 +401,14 @@ export class SplatAccumulator { base, count, renderer, + useRelativeCenter, }: { generator?: GsplatGenerator; covGenerator?: CovSplatGenerator; base: number; count: number; renderer: THREE.WebGLRenderer; + useRelativeCenter?: boolean; }) { if (!this.target) { throw new Error("Target must be initialized with ensureGenerate"); @@ -385,6 +420,7 @@ export class SplatAccumulator { const { program, material } = this.prepareProgramMaterial( generator, covGenerator, + useRelativeCenter, ); program.update(); @@ -571,7 +607,14 @@ export class SplatAccumulator { for (const { node, base, count } of this.mapping) { const { generator, covGenerator } = node; if ((generator || covGenerator) && count > 0) { - this.generate({ generator, covGenerator, base, count, renderer }); + this.generate({ + generator, + covGenerator, + base, + count, + renderer, + useRelativeCenter: node.useRelativeCenter, + }); } } }, diff --git a/src/SplatGenerator.ts b/src/SplatGenerator.ts index de778c7e..01586e8c 100644 --- a/src/SplatGenerator.ts +++ b/src/SplatGenerator.ts @@ -271,6 +271,7 @@ export class SplatGenerator extends THREE.Object3D { frameUpdate?: (context: FrameUpdateContext) => void; version: number; mappingVersion: number; + useRelativeCenter: boolean; constructor({ numSplats, @@ -298,6 +299,7 @@ export class SplatGenerator extends THREE.Object3D { this.frameUpdate = update; this.version = 0; this.mappingVersion = 0; + this.useRelativeCenter = false; if (construct) { const constructed = construct(this); diff --git a/src/SplatMesh.ts b/src/SplatMesh.ts index b1027f67..d67f9e5b 100644 --- a/src/SplatMesh.ts +++ b/src/SplatMesh.ts @@ -149,6 +149,11 @@ export type SplatMeshOptions = { paged?: boolean | PagedSplats | SplatPager; }; +// SplatMeshContext provides dyno uniforms used to build the Gsplat processing +// pipeline. A SplatMesh exposes two contexts: +// - context: stable, absolute-world semantics for public API consumers +// - renderContext: internal render-space semantics, which may become +// camera-relative when useRelativeCenter is enabled export type SplatMeshContext = { transform: SplatTransformer; viewToWorld: SplatTransformer; @@ -241,16 +246,23 @@ export class SplatMesh extends SplatGenerator { // Global opacity multiplier for all splats in the mesh. (default: 1) opacity = 1; - // A SplatMeshContext consisting of useful scene and object dyno uniforms that can - // be used to in the Gsplat processing pipeline, for example via objectModifier and - // worldModifier. (created on construction) + // Public dyno context with absolute-world semantics. Use this when your + // modifier or generator expects stable world transforms or absolute-space + // reasoning outside the render path. context: SplatMeshContext; + // Render-space dyno context used by the built-in generation/render path. + // When useRelativeCenter is enabled, these transforms may become + // camera-relative to improve precision in large worlds. + // Use this for modifiers that need to stay aligned with the render path, + // such as view-space or depth-based effects. + renderContext: SplatMeshContext; onFrame?: ({ mesh, time, deltaTime, }: { mesh: SplatMesh; time: number; deltaTime: number }) => void; generatorDirty = true; + private generatorUsesRelativeCenter = false; objectModifiers?: GsplatModifier[]; worldModifiers?: GsplatModifier[]; @@ -366,6 +378,23 @@ export class SplatMesh extends SplatGenerator { key: "lodIndices", }), }; + this.renderContext = { + transform: new SplatTransformer(), + viewToWorld: new SplatTransformer(), + worldToView: new SplatTransformer(), + viewToObject: new SplatTransformer(), + covTransform: new CovSplatTransformer(), + covViewToWorld: new CovSplatTransformer(), + covWorldToView: new CovSplatTransformer(), + covViewToObject: new CovSplatTransformer(), + recolor: this.context.recolor, + time: this.context.time, + deltaTime: this.context.deltaTime, + numSplats: this.context.numSplats, + splats: this.context.splats, + enableLod: this.context.enableLod, + lodIndices: this.context.lodIndices, + }; this.covSplats = options.covSplats ?? false; if (this.covSplats && !this.extSplats) { @@ -845,11 +874,20 @@ export class SplatMesh extends SplatGenerator { const splats = this.splats ?? this.packedSplats ?? this.extSplats; if (splats) { this.context.splats = splats; + this.renderContext.splats = splats; } this.numSplats = this.context.splats.getNumSplats(); let updated = false; + if (this.generatorUsesRelativeCenter !== this.useRelativeCenter) { + this.generatorUsesRelativeCenter = this.useRelativeCenter; + this.generatorDirty = true; + } + + const canUpdateViewToObject = + this.enableViewToObject || this.context.splats.hasRgbDir(); + if (!this.covSplats) { if (this.context.transform.update(this)) { updated = true; @@ -878,7 +916,7 @@ export class SplatMesh extends SplatGenerator { const viewToObjectMatrix = worldToObject.multiply(viewToWorld); if ( this.context.viewToObject.updateFromMatrix(viewToObjectMatrix) && - (this.enableViewToObject || this.context.splats.hasRgbDir()) + canUpdateViewToObject ) { // Only trigger update if we have view-dependent spherical harmonics updated = true; @@ -906,13 +944,95 @@ export class SplatMesh extends SplatGenerator { const viewToObjectMatrix = worldToObject.multiply(viewToWorld); if ( this.context.covViewToObject.updateFromMatrix(viewToObjectMatrix) && - (this.enableViewToObject || this.context.splats.hasRgbDir()) + canUpdateViewToObject ) { // Only trigger update if we have view-dependent spherical harmonics updated = true; } } + if (this.useRelativeCenter) { + // Large world coordinates mode: compute transforms relative to camera + // position to avoid floating-point precision issues at large coordinates. + const cameraWorldPosition = new THREE.Vector3().setFromMatrixPosition( + viewToWorld, + ); + const relativeViewToWorld = viewToWorld.clone(); + relativeViewToWorld.setPosition(0, 0, 0); + const relativeWorldToView = relativeViewToWorld.clone().invert(); + + this.updateMatrixWorld(); + const relativeObjectToWorld = this.matrixWorld.clone(); + const objectWorldPosition = new THREE.Vector3().setFromMatrixPosition( + relativeObjectToWorld, + ); + relativeObjectToWorld.setPosition( + objectWorldPosition.x - cameraWorldPosition.x, + objectWorldPosition.y - cameraWorldPosition.y, + objectWorldPosition.z - cameraWorldPosition.z, + ); + + if (!this.covSplats) { + if ( + this.renderContext.transform.updateFromMatrix(relativeObjectToWorld) + ) { + updated = true; + } + if ( + this.renderContext.viewToWorld.updateFromMatrix(relativeViewToWorld) && + this.enableViewToWorld + ) { + updated = true; + } + if ( + this.renderContext.worldToView.updateFromMatrix(relativeWorldToView) && + this.enableWorldToView + ) { + updated = true; + } + const relativeViewToObject = relativeObjectToWorld + .clone() + .invert() + .multiply(relativeViewToWorld); + if ( + this.renderContext.viewToObject.updateFromMatrix(relativeViewToObject) && + canUpdateViewToObject + ) { + updated = true; + } + } else { + if ( + this.renderContext.covTransform.updateFromMatrix(relativeObjectToWorld) + ) { + updated = true; + } + if ( + this.renderContext.covViewToWorld.updateFromMatrix(relativeViewToWorld) && + this.enableViewToWorld + ) { + updated = true; + } + if ( + this.renderContext.covWorldToView.updateFromMatrix(relativeWorldToView) && + this.enableWorldToView + ) { + updated = true; + } + const relativeViewToObject = relativeObjectToWorld + .clone() + .invert() + .multiply(relativeViewToWorld); + if ( + this.renderContext.covViewToObject.updateFromMatrix(relativeViewToObject) && + canUpdateViewToObject + ) { + updated = true; + } + } + } else { + this.renderContext.splats = this.context.splats; + } + const newRecolor = new THREE.Vector4( this.recolor.r, this.recolor.g, @@ -977,6 +1097,7 @@ export class SplatMesh extends SplatGenerator { if (this.context.enableLod.value && lodSplats) { this.context.splats = lodSplats; + this.renderContext.splats = lodSplats; this.numSplats = lodIndices?.numSplats ?? 0; } @@ -988,7 +1109,9 @@ export class SplatMesh extends SplatGenerator { } if (this.generatorDirty) { - this.constructGenerator(this.context); + this.constructGenerator( + this.useRelativeCenter ? this.renderContext : this.context, + ); this.generatorDirty = false; updated = true; } diff --git a/src/modifiers/depthColor.ts b/src/modifiers/depthColor.ts index 9d788ddf..42017ca0 100644 --- a/src/modifiers/depthColor.ts +++ b/src/modifiers/depthColor.ts @@ -45,8 +45,10 @@ export function setDepthColor( const dynoMinDepth = dynoConst("float", minDepth); const dynoMaxDepth = dynoConst("float", maxDepth); const dynoReverse = dynoConst("bool", reverse ?? false); + // Depth coloring must follow the active render-space transform so that + // large-world relative rendering still produces correct view-space depth. splats.worldModifier = makeDepthColorModifier( - splats.context.worldToView, + splats.renderContext.worldToView, dynoMinDepth, dynoMaxDepth, dynoReverse, diff --git a/src/modifiers/normalColor.ts b/src/modifiers/normalColor.ts index ac095413..a70e52d1 100644 --- a/src/modifiers/normalColor.ts +++ b/src/modifiers/normalColor.ts @@ -41,6 +41,10 @@ export function makeNormalColorModifier(splatToView: SplatTransformer) { export function setWorldNormalColor(splats: SplatMesh) { splats.enableWorldToView = true; - splats.worldModifier = makeNormalColorModifier(splats.context.worldToView); + // Normal coloring must follow the active render-space transform so that + // large-world relative rendering still produces correct view-space normals. + splats.worldModifier = makeNormalColorModifier( + splats.renderContext.worldToView, + ); splats.updateGenerator(); }