fix: add gaussian splatting shaders

examples/demos/fps-controller.ts

@@ -25,6 +25,10 @@ import {
   FpsCameraBundle,
   FpsCameraController,
   FpsCameraPlugin,
+  VisibleEntities,
+  Visibility,
+  InheritedVisibility,
+  ViewVisibility,
 } from '../../src';
 import { loadImage } from '../utils/image';
 // @ts-ignore
@@ -58,6 +62,10 @@ export async function render($canvas: HTMLCanvasElement, gui: lil.GUI) {
           LookTransform,
           FpsCameraController,
           Smoother,
+          VisibleEntities,
+          Visibility,
+          InheritedVisibility,
+          ViewVisibility,
         ).write,
     );
 

examples/demos/gaussian-splatting.ts

@@ -22,21 +22,34 @@ import {
   TonemappingMethod,
   DebandDither,
   GaussianSplattingPlugin,
+  Gaussian,
+  SphericalHarmonicCoefficients,
+  GaussianCloud,
+  VisibleEntities,
+  Visibility,
+  InheritedVisibility,
+  ViewVisibility,
 } from '../../src';
 import { loadImage } from '../utils/image';
 // @ts-ignore
 import glsl_wgsl_compiler_bg from '../public/glsl_wgsl_compiler_bg.wasm?url';
 import { parse } from '@loaders.gl/core';
 import { PLYLoader } from '@loaders.gl/ply';
+import {
+  PositionVisibility,
+  Rotation,
+  ScaleOpacity,
+} from '../../src/components/gaussian-splatting/f32';
+
+const MAX_SIZE_VARIANCE = 5.0;
 
 /**
  * @see https://bevyengine.org/learn/book/getting-started/ecs/
  */
 export async function render($canvas: HTMLCanvasElement, gui: lil.GUI) {
   let camera: Entity;
 
-  const data = await parse(fetch('/icecream.ply'), PLYLoader);
-  console.log(data);
+  const { header, attributes } = await parse(fetch('/icecream.ply'), PLYLoader);
 
   class StartUpSystem extends System {
     commands = new Commands(this);
@@ -54,6 +67,10 @@ export async function render($canvas: HTMLCanvasElement, gui: lil.GUI) {
           ColorGrading,
           Tonemapping,
           DebandDither,
+          VisibleEntities,
+          Visibility,
+          InheritedVisibility,
+          ViewVisibility,
         ).write,
     );
 
@@ -72,6 +89,69 @@ export async function render($canvas: HTMLCanvasElement, gui: lil.GUI) {
         )
         .entity.hold();
 
+      const gaussians: Gaussian[] = [];
+      for (let i = 0; i < (header?.vertexCount || 0); i++) {
+        const x = attributes.POSITION.value[i * 3 + 0];
+        const y = attributes.POSITION.value[i * 3 + 1];
+        const z = attributes.POSITION.value[i * 3 + 2];
+        const f_dc_0 = attributes.f_dc_0.value[i];
+        const f_dc_1 = attributes.f_dc_1.value[i];
+        const f_dc_2 = attributes.f_dc_2.value[i];
+        const scale_0 = attributes.scale_0.value[i];
+        const scale_1 = attributes.scale_1.value[i];
+        const scale_2 = attributes.scale_2.value[i];
+        const opacity = attributes.opacity.value[i];
+        const rot_0 = attributes.rot_0.value[i];
+        const rot_1 = attributes.rot_1.value[i];
+        const rot_2 = attributes.rot_2.value[i];
+        const rot_3 = attributes.rot_3.value[i];
+
+        const gaussian = new Gaussian({
+          rotation: new Rotation({ rotation: [rot_0, rot_1, rot_2, rot_3] }),
+          position_visibility: new PositionVisibility({
+            position: [x, y, z],
+          }),
+          scale_opacity: new ScaleOpacity({
+            scale: [scale_0, scale_1, scale_2],
+            opacity,
+          }),
+          spherical_harmonic: new SphericalHarmonicCoefficients({
+            coefficients: [f_dc_0, f_dc_1, f_dc_2],
+          }),
+        });
+        gaussians.push(gaussian);
+
+        gaussian.position_visibility.visibility = 1.0;
+        let mean_scale =
+          (gaussian.scale_opacity.scale[0] +
+            gaussian.scale_opacity.scale[1] +
+            gaussian.scale_opacity.scale[2]) /
+          3.0;
+        for (let i = 0; i < 3; i++) {
+          gaussian.scale_opacity.scale[i] = Math.exp(
+            Math.min(
+              Math.max(
+                gaussian.scale_opacity.scale[i],
+                mean_scale - MAX_SIZE_VARIANCE,
+              ),
+              mean_scale + MAX_SIZE_VARIANCE,
+            ),
+          );
+        }
+
+        const norm = Math.sqrt(
+          new Array(4)
+            .map((i) => Math.pow(gaussian.rotation.rotation[i], 2.0))
+            .reduce((prev, cur) => prev + cur, 0),
+        );
+        for (let i = 0; i < 4; i++) {
+          gaussian.rotation.rotation[i] /= norm;
+        }
+      }
+
+      const cloud = GaussianCloud.from_gaussians(gaussians);
+      console.log(cloud);
+
       // const mesh = Mesh.from(new Cube(1));
       // const material = new Material({
       //   base_color_texture: baseColorImage,

examples/demos/light.ts

@@ -32,6 +32,11 @@ import {
   AmbientLight,
   CascadeShadowConfigBuilder,
   Plane,
+  CascadesVisibleEntities,
+  Visibility,
+  VisibleEntities,
+  InheritedVisibility,
+  ViewVisibility,
 } from '../../src';
 import { loadImage } from '../utils/image';
 // @ts-ignore
@@ -72,6 +77,11 @@ export async function render($canvas: HTMLCanvasElement, gui: lil.GUI) {
           CascadesFrusta,
           Cascades,
           CascadeShadowConfig,
+          CascadesVisibleEntities,
+          Visibility,
+          InheritedVisibility,
+          ViewVisibility,
+          VisibleEntities,
         ).write,
     );
 

examples/demos/orbit-controller.ts

@@ -30,6 +30,10 @@ import {
   OrbitCameraBundle,
   OrbitCameraController,
   OrbitCameraPlugin,
+  VisibleEntities,
+  Visibility,
+  InheritedVisibility,
+  ViewVisibility,
 } from '../../src';
 import { loadImage } from '../utils/image';
 // @ts-ignore
@@ -65,6 +69,10 @@ export async function render($canvas: HTMLCanvasElement, gui: lil.GUI) {
           LookTransform,
           OrbitCameraController,
           Smoother,
+          Visibility,
+          VisibleEntities,
+          InheritedVisibility,
+          ViewVisibility,
         ).write,
     );
 

src/components/camera/Camera3dBundle.ts

@@ -6,30 +6,25 @@ import { Bundle } from '../Bundle';
 import { ComputedCameraValues } from './ComputedCameraValues';
 import { Frustum } from '../primitive/Frustum';
 import { Tonemapping } from '../pipeline';
-import { ColorGrading, DebandDither } from '../render';
+import { ColorGrading, DebandDither, VisibleEntities } from '../render';
 
 export class Camera3dBundle extends Bundle {
   camera: Camera;
-
   computed: ComputedCameraValues;
-
   projection: Perspective | Orthographic;
-
+  visible_entities: VisibleEntities;
+  frustum: Frustum;
   transform: Transform;
-
   tonemapping: Tonemapping;
-
   dither: DebandDither;
-
   color_grading: ColorGrading;
 
-  frustum: Frustum;
-
   constructor(
     options?: Partial<{
       camera: Camera;
       computed: ComputedCameraValues;
       projection: Perspective | Orthographic;
+      visible_entities: VisibleEntities;
       transform: Transform;
       global_transform: GlobalTransform;
       tonemapping: Tonemapping;
@@ -45,6 +40,7 @@ export class Camera3dBundle extends Bundle {
       computed = new ComputedCameraValues(),
       transform,
       projection = new Perspective(),
+      visible_entities = new VisibleEntities(),
       tonemapping = new Tonemapping(),
       color_grading = new ColorGrading(),
       dither = new DebandDither(),
@@ -55,6 +51,7 @@ export class Camera3dBundle extends Bundle {
     this.computed = computed;
     this.transform = transform;
     this.projection = projection;
+    this.visible_entities = visible_entities;
     this.tonemapping = tonemapping;
     this.color_grading = color_grading;
     this.dither = dither;

src/components/light/CascadesVisibleEntities.ts

@@ -0,0 +1,10 @@
+import { Entity, field } from '@lastolivegames/becsy';
+import { VisibleEntities } from '../render';
+
+export class CascadesVisibleEntities {
+  @field.object declare entities: Map<Entity, VisibleEntities[]>;
+
+  constructor() {
+    this.entities = new Map();
+  }
+}

src/components/light/DirectionalLightBundle.ts

@@ -1,35 +1,34 @@
 import { Bundle } from '../Bundle';
 import { CascadesFrusta } from '../primitive';
+import { InheritedVisibility, ViewVisibility, Visibility } from '../render';
 import { Transform } from '../transform';
 import { CascadeShadowConfig } from './CascadeShadowConfig';
 import { Cascades } from './Cascades';
+import { CascadesVisibleEntities } from './CascadesVisibleEntities';
 import { DirectionalLight } from './DirectionalLight';
 
 export class DirectionalLightBundle extends Bundle {
   directional_light: DirectionalLight;
   frusta: CascadesFrusta;
   cascades: Cascades;
   cascade_shadow_config: CascadeShadowConfig;
-  // pub visible_entities: CascadesVisibleEntities,
+  visible_entities: CascadesVisibleEntities;
   transform: Transform;
-  // /// Enables or disables the light
-  // pub visibility: Visibility,
-  // /// Inherited visibility of an entity.
-  // pub inherited_visibility: InheritedVisibility,
-  // /// Algorithmically-computed indication of whether an entity is visible and should be extracted for rendering
-  // pub view_visibility: ViewVisibility,
+  visibility: Visibility;
+  inherited_visibility: InheritedVisibility;
+  view_visibility: ViewVisibility;
 
   constructor(
     options?: Partial<{
       directional_light: DirectionalLight;
       frusta: CascadesFrusta;
       cascades: Cascades;
       cascade_shadow_config: CascadeShadowConfig;
-      // visible_entities: CascadesVisibleEntities,
+      visible_entities: CascadesVisibleEntities;
       transform: Transform;
-      // visibility: Visibility,
-      // inherited_visibility: InheritedVisibility,
-      // view_visibility: ViewVisibility,
+      visibility: Visibility;
+      inherited_visibility: InheritedVisibility;
+      view_visibility: ViewVisibility;
     }>,
   ) {
     super();
@@ -39,21 +38,21 @@ export class DirectionalLightBundle extends Bundle {
       frusta = new CascadesFrusta(),
       cascades = new Cascades(),
       cascade_shadow_config = new CascadeShadowConfig(),
-      // visible_entities = new CascadesVisibleEntities(),
+      visible_entities = new CascadesVisibleEntities(),
       transform,
-      // visibility = new Visibility(),
-      // inherited_visibility = new InheritedVisibility(),
-      // view_visibility = new ViewVisibility(),
+      visibility = new Visibility(),
+      inherited_visibility = new InheritedVisibility(),
+      view_visibility = new ViewVisibility(),
     } = options || {};
 
     this.directional_light = directional_light;
     this.frusta = frusta;
     this.cascades = cascades;
     this.cascade_shadow_config = cascade_shadow_config;
-    // this.visible_entities = visible_entities;
+    this.visible_entities = visible_entities;
     this.transform = transform;
-    // this.visibility = visibility;
-    // this.inherited_visibility = inherited_visibility;
-    // this.view_visibility = view_visibility;
+    this.visibility = visibility;
+    this.inherited_visibility = inherited_visibility;
+    this.view_visibility = view_visibility;
   }
 }

src/components/light/index.ts

@@ -7,4 +7,5 @@ export * from './DirectionalLightBundle';
 export * from './Clusters';
 export * from './ClusterConfig';
 export * from './CascadeShadowConfig';
+export * from './CascadesVisibleEntities';
 export * from './Cascades';

src/components/pbr/PbrBundle.ts

@@ -40,9 +40,9 @@ export class PbrBundle extends Bundle {
       mesh,
       material,
       transform,
-      visibility,
-      inherited_visibility,
-      view_visibility,
+      visibility = new Visibility(),
+      inherited_visibility = new InheritedVisibility(),
+      view_visibility = new ViewVisibility(),
     } = options || {};
     this.mesh = mesh;
     this.material = material;

src/components/primitive/CascadesFrusta.ts

@@ -3,4 +3,8 @@ import { Frustum } from './Frustum';
 
 export class CascadesFrusta {
   @field.object declare frusta: Map<Entity, Frustum[]>;
+
+  constructor() {
+    this.frusta = new Map();
+  }
 }

src/components/render/RenderLayers.ts

@@ -1,3 +1,5 @@
+import { field } from '@lastolivegames/becsy';
+
 /// An identifier for a rendering layer.
 export type Layer = number;
 
@@ -45,7 +47,11 @@ export class RenderLayers {
   //   layers.iter().copied().collect()
   // }
 
-  constructor(private layer = 0) {}
+  @field.uint8 declare layer: number;
+
+  constructor(layer = 0) {
+    this.layer = layer;
+  }
 
   /**
    * Add the given layer.

src/components/transform/GlobalTransform.ts

@@ -13,7 +13,7 @@ import { Transform } from './Transform';
 export class GlobalTransform extends Affine3 {
   static copy(rhs: GlobalTransform) {
     return new GlobalTransform(
-      Mat3.copy(rhs.affine()),
+      Mat3.copy(rhs.matrix3),
       Vec3.copy(rhs.translation),
     );
   }
@@ -43,12 +43,16 @@ export class GlobalTransform extends Affine3 {
    * Returns the 3d affine transformation matrix as an [`Affine3A`].
    */
   affine() {
-    return this.matrix3;
+    return this;
   }
 
   from(transform: Transform) {
     const { matrix3, translation } = transform.compute_affine();
     this.matrix3 = matrix3;
     this.translation = translation;
   }
+
+  radius_vec3(extents: Vec3) {
+    return this.matrix3.mul_vec3(extents)._length();
+  }
 }