Skip to content

Add camera-relative splat rendering support for large world coordinates #291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
WilliamLiu-1997 wants to merge 1 commit into
base: v2.0.0-preview
Choose a base branch
from
Open

Add camera-relative splat rendering support for large world coordinates #291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions src/SparkRenderer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -672,10 +672,9 @@ export class SparkRenderer extends THREE.Mesh {
const geometry = this.geometry as SplatGeometry;
geometry.instanceCount = spark.activeSplats;

const accumToWorld = new THREE.Matrix4();
if (!this.display.extSplats) {
accumToWorld.makeTranslation(spark.display.viewOrigin);
}
const accumToWorld = new THREE.Matrix4().makeTranslation(
spark.display.viewOrigin,
);
const cameraToWorld = camera.matrixWorld.clone();
const worldToCamera = cameraToWorld.invert();
const accumToCamera = worldToCamera.multiply(accumToWorld);
Expand Down
65 changes: 54 additions & 11 deletions src/SplatAccumulator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,12 +210,28 @@ export class SplatAccumulator {
prepareProgramMaterial(
generator?: GsplatGenerator,
covGenerator?: CovSplatGenerator,
useRelativeCenter = false,
) {
const theGenerator = generator ?? covGenerator;
if (!theGenerator) {
throw new Error("Either generator or covGenerator must be provided");
}

// Invalidate cached program if the relative center mode changed
const cachedRelative =
SplatAccumulator.generatorRelativeCenter.get(theGenerator);
if (cachedRelative !== undefined && cachedRelative !== useRelativeCenter) {
SplatAccumulator.generatorProgram.delete(theGenerator);
}
SplatAccumulator.generatorRelativeCenter.set(
theGenerator,
useRelativeCenter,
);

const viewCenter = useRelativeCenter
? SplatAccumulator.relativeCenterZero
: SplatAccumulator.viewCenterUniform;

let program = SplatAccumulator.generatorProgram.get(theGenerator);
if (!program) {
const graph = dynoBlock(
Expand All @@ -232,29 +248,39 @@ export class SplatAccumulator {
if (this.extSplats) {
if (!this.covSplats) {
if (generator) {
const output = outputExtendedSplat(generator.outputs.gsplat);
const split = splitGsplat(generator.outputs.gsplat).outputs;
const gsplat = combineGsplat({
gsplat: generator.outputs.gsplat,
center: sub(split.center, viewCenter),
});
const output = outputExtendedSplat(gsplat);
roots.push(output);
} else {
throw new Error("Generator must be provided");
}
} else {
let covsplat: DynoVal<typeof CovSplat>;
if (covGenerator) {
const output = outputExtCovSplat(covGenerator.outputs.covsplat);
roots.push(output);
covsplat = covGenerator.outputs.covsplat;
} else if (generator) {
const covsplat = gsplatToCovSplat(generator.outputs.gsplat);
const output = outputExtCovSplat(covsplat);
roots.push(output);
covsplat = gsplatToCovSplat(generator.outputs.gsplat);
} else {
throw new Error("Generator must be provided");
}
const split = splitCovSplat(covsplat).outputs;
const centeredCovSplat = combineCovSplat({
covsplat,
center: sub(split.center, viewCenter),
});
const output = outputExtCovSplat(centeredCovSplat);
roots.push(output);
}
} else {
if (!this.covSplats) {
if (generator) {
const centerSubView = sub(
splitGsplat(generator.outputs.gsplat).outputs.center,
SplatAccumulator.viewCenterUniform,
viewCenter,
);
// Use expanded LoD opacity encoding
const halfAlpha = mul(
Expand Down Expand Up @@ -285,7 +311,7 @@ export class SplatAccumulator {
}
const centerSubView = sub(
splitCovSplat(covsplat).outputs.center,
SplatAccumulator.viewCenterUniform,
viewCenter,
);
const halfAlpha = mul(
splitCovSplat(covsplat).outputs.opacity,
Expand All @@ -309,7 +335,7 @@ export class SplatAccumulator {
if (generator) {
const outputDepth = outputSplatDepth(
generator.outputs.gsplat,
SplatAccumulator.viewCenterUniform,
viewCenter,
SplatAccumulator.viewDirUniform,
SplatAccumulator.sortRadialUniform,
);
Expand All @@ -318,7 +344,7 @@ export class SplatAccumulator {
if (covGenerator) {
const outputDepth = outputCovSplatDepth(
covGenerator.outputs.covsplat,
SplatAccumulator.viewCenterUniform,
viewCenter,
SplatAccumulator.viewDirUniform,
SplatAccumulator.sortRadialUniform,
);
Expand Down Expand Up @@ -358,6 +384,13 @@ export class SplatAccumulator {
GsplatGenerator | CovSplatGenerator,
DynoProgram
>();
static generatorRelativeCenter = new Map<
GsplatGenerator | CovSplatGenerator,
boolean
>();
static relativeCenterZero = new DynoVec3({
value: new THREE.Vector3(),
});
static fullScreenQuad = new FullScreenQuad(
new THREE.RawShaderMaterial({ visible: false }),
);
Expand All @@ -368,12 +401,14 @@ export class SplatAccumulator {
base,
count,
renderer,
useRelativeCenter,
}: {
generator?: GsplatGenerator;
covGenerator?: CovSplatGenerator;
base: number;
count: number;
renderer: THREE.WebGLRenderer;
useRelativeCenter?: boolean;
}) {
if (!this.target) {
throw new Error("Target must be initialized with ensureGenerate");
Expand All @@ -385,6 +420,7 @@ export class SplatAccumulator {
const { program, material } = this.prepareProgramMaterial(
generator,
covGenerator,
useRelativeCenter,
);
program.update();

Expand Down Expand Up @@ -571,7 +607,14 @@ export class SplatAccumulator {
for (const { node, base, count } of this.mapping) {
const { generator, covGenerator } = node;
if ((generator || covGenerator) && count > 0) {
this.generate({ generator, covGenerator, base, count, renderer });
this.generate({
generator,
covGenerator,
base,
count,
renderer,
useRelativeCenter: node.useRelativeCenter,
});
}
}
},
Expand Down
2 changes: 2 additions & 0 deletions src/SplatGenerator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ export class SplatGenerator extends THREE.Object3D {
frameUpdate?: (context: FrameUpdateContext) => void;
version: number;
mappingVersion: number;
useRelativeCenter: boolean;

constructor({
numSplats,
Expand Down Expand Up @@ -298,6 +299,7 @@ export class SplatGenerator extends THREE.Object3D {
this.frameUpdate = update;
this.version = 0;
this.mappingVersion = 0;
this.useRelativeCenter = false;

if (construct) {
const constructed = construct(this);
Expand Down
135 changes: 129 additions & 6 deletions src/SplatMesh.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,11 @@ export type SplatMeshOptions = {
paged?: boolean | PagedSplats | SplatPager;
};

// SplatMeshContext provides dyno uniforms used to build the Gsplat processing
// pipeline. A SplatMesh exposes two contexts:
// - context: stable, absolute-world semantics for public API consumers
// - renderContext: internal render-space semantics, which may become
// camera-relative when useRelativeCenter is enabled
export type SplatMeshContext = {
transform: SplatTransformer;
viewToWorld: SplatTransformer;
Expand Down Expand Up @@ -241,16 +246,23 @@ export class SplatMesh extends SplatGenerator {
// Global opacity multiplier for all splats in the mesh. (default: 1)
opacity = 1;

// A SplatMeshContext consisting of useful scene and object dyno uniforms that can
// be used to in the Gsplat processing pipeline, for example via objectModifier and
// worldModifier. (created on construction)
// Public dyno context with absolute-world semantics. Use this when your
// modifier or generator expects stable world transforms or absolute-space
// reasoning outside the render path.
context: SplatMeshContext;
// Render-space dyno context used by the built-in generation/render path.
// When useRelativeCenter is enabled, these transforms may become
// camera-relative to improve precision in large worlds.
// Use this for modifiers that need to stay aligned with the render path,
// such as view-space or depth-based effects.
renderContext: SplatMeshContext;
onFrame?: ({
mesh,
time,
deltaTime,
}: { mesh: SplatMesh; time: number; deltaTime: number }) => void;
generatorDirty = true;
private generatorUsesRelativeCenter = false;

objectModifiers?: GsplatModifier[];
worldModifiers?: GsplatModifier[];
Expand Down Expand Up @@ -366,6 +378,23 @@ export class SplatMesh extends SplatGenerator {
key: "lodIndices",
}),
};
this.renderContext = {
transform: new SplatTransformer(),
viewToWorld: new SplatTransformer(),
worldToView: new SplatTransformer(),
viewToObject: new SplatTransformer(),
covTransform: new CovSplatTransformer(),
covViewToWorld: new CovSplatTransformer(),
covWorldToView: new CovSplatTransformer(),
covViewToObject: new CovSplatTransformer(),
recolor: this.context.recolor,
time: this.context.time,
deltaTime: this.context.deltaTime,
numSplats: this.context.numSplats,
splats: this.context.splats,
enableLod: this.context.enableLod,
lodIndices: this.context.lodIndices,
};

this.covSplats = options.covSplats ?? false;
if (this.covSplats && !this.extSplats) {
Expand Down Expand Up @@ -845,11 +874,20 @@ export class SplatMesh extends SplatGenerator {
const splats = this.splats ?? this.packedSplats ?? this.extSplats;
if (splats) {
this.context.splats = splats;
this.renderContext.splats = splats;
}
this.numSplats = this.context.splats.getNumSplats();

let updated = false;

if (this.generatorUsesRelativeCenter !== this.useRelativeCenter) {
this.generatorUsesRelativeCenter = this.useRelativeCenter;
this.generatorDirty = true;
}

const canUpdateViewToObject =
this.enableViewToObject || this.context.splats.hasRgbDir();

if (!this.covSplats) {
if (this.context.transform.update(this)) {
updated = true;
Expand Down Expand Up @@ -878,7 +916,7 @@ export class SplatMesh extends SplatGenerator {
const viewToObjectMatrix = worldToObject.multiply(viewToWorld);
if (
this.context.viewToObject.updateFromMatrix(viewToObjectMatrix) &&
(this.enableViewToObject || this.context.splats.hasRgbDir())
canUpdateViewToObject
) {
// Only trigger update if we have view-dependent spherical harmonics
updated = true;
Expand Down Expand Up @@ -906,13 +944,95 @@ export class SplatMesh extends SplatGenerator {
const viewToObjectMatrix = worldToObject.multiply(viewToWorld);
if (
this.context.covViewToObject.updateFromMatrix(viewToObjectMatrix) &&
(this.enableViewToObject || this.context.splats.hasRgbDir())
canUpdateViewToObject
) {
// Only trigger update if we have view-dependent spherical harmonics
updated = true;
}
}

if (this.useRelativeCenter) {
// Large world coordinates mode: compute transforms relative to camera
// position to avoid floating-point precision issues at large coordinates.
const cameraWorldPosition = new THREE.Vector3().setFromMatrixPosition(
viewToWorld,
);
const relativeViewToWorld = viewToWorld.clone();
relativeViewToWorld.setPosition(0, 0, 0);
const relativeWorldToView = relativeViewToWorld.clone().invert();

this.updateMatrixWorld();
const relativeObjectToWorld = this.matrixWorld.clone();
const objectWorldPosition = new THREE.Vector3().setFromMatrixPosition(
relativeObjectToWorld,
);
relativeObjectToWorld.setPosition(
objectWorldPosition.x - cameraWorldPosition.x,
objectWorldPosition.y - cameraWorldPosition.y,
objectWorldPosition.z - cameraWorldPosition.z,
);

if (!this.covSplats) {
if (
this.renderContext.transform.updateFromMatrix(relativeObjectToWorld)
) {
updated = true;
}
if (
this.renderContext.viewToWorld.updateFromMatrix(relativeViewToWorld) &&
this.enableViewToWorld
) {
updated = true;
}
if (
this.renderContext.worldToView.updateFromMatrix(relativeWorldToView) &&
this.enableWorldToView
) {
updated = true;
}
const relativeViewToObject = relativeObjectToWorld
.clone()
.invert()
.multiply(relativeViewToWorld);
if (
this.renderContext.viewToObject.updateFromMatrix(relativeViewToObject) &&
canUpdateViewToObject
) {
updated = true;
}
} else {
if (
this.renderContext.covTransform.updateFromMatrix(relativeObjectToWorld)
) {
updated = true;
}
if (
this.renderContext.covViewToWorld.updateFromMatrix(relativeViewToWorld) &&
this.enableViewToWorld
) {
updated = true;
}
if (
this.renderContext.covWorldToView.updateFromMatrix(relativeWorldToView) &&
this.enableWorldToView
) {
updated = true;
}
const relativeViewToObject = relativeObjectToWorld
.clone()
.invert()
.multiply(relativeViewToWorld);
if (
this.renderContext.covViewToObject.updateFromMatrix(relativeViewToObject) &&
canUpdateViewToObject
) {
updated = true;
}
}
} else {
this.renderContext.splats = this.context.splats;
}

const newRecolor = new THREE.Vector4(
this.recolor.r,
this.recolor.g,
Expand Down Expand Up @@ -977,6 +1097,7 @@ export class SplatMesh extends SplatGenerator {

if (this.context.enableLod.value && lodSplats) {
this.context.splats = lodSplats;
this.renderContext.splats = lodSplats;
this.numSplats = lodIndices?.numSplats ?? 0;
}

Expand All @@ -988,7 +1109,9 @@ export class SplatMesh extends SplatGenerator {
}

if (this.generatorDirty) {
this.constructGenerator(this.context);
this.constructGenerator(
this.useRelativeCenter ? this.renderContext : this.context,
);
this.generatorDirty = false;
updated = true;
}
Expand Down
Loading